WIP: refactor for Java bindings

.github/workflows/build_jar.yml

@@ -0,0 +1,81 @@
+name: Build Java JAR
+
+on: [push, pull_request, workflow_dispatch]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build_jni:
+    name: jni on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        include:
+          - os: ubuntu-latest
+            outdir: linux_64
+          - os: windows-latest
+            outdir: windows_64
+          - os: macos-latest
+            outdir: osx_64
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          # stable doesn't have --out-dir
+          toolchain: nightly
+          override: true
+
+      - name: Build
+        working-directory: ./jni
+        # TODO: 32bit vs 64bit?
+        # https://github.com/scijava/native-lib-loader
+        run: cargo build --release -Z unstable-options --out-dir ../build/natives/${{ matrix.outdir }}/
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: natives
+          path: ./build/natives/*
+
+  build_java:
+    name: java
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    needs: [build_jni]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Load outputs
+        uses: actions/download-artifact@v3
+        with:
+          name: natives
+          path: natives
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'microsoft'
+          architecture: x64
+          cache: maven
+
+      - name: Build with Maven
+        working-directory: ./java
+        run: mvn --batch-mode package failsafe:integration-test
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: java
+          path: ./java/target/*.jar
+
+      # TODO: publish to maven (only from ubuntu)
+

Cargo.toml

@@ -1,21 +1,15 @@
-[package]
-name = "tiktoken"
-version = "0.2.0"
-edition = "2021"
-rust-version = "1.57.0"
+[workspace]
 
-[lib]
-name = "_tiktoken"
-crate-type = ["cdylib"]
-
-[dependencies]
-pyo3 = { version = "0.17.3", features = ["extension-module"] }
-
-# tiktoken dependencies
-fancy-regex = "0.10.0"
-regex = "1.7.0"
-rustc-hash = "1.1.0"
-bstr = "1.0.1"
+members = [
+    "core",
+    "python",
+    "jni",
+]
 
 [profile.release]
 incremental = true
+opt-level = 'z'     # Optimize for size
+lto = true          # Enable link-time optimization
+codegen-units = 1   # Reduce number of codegen units to increase optimizations
+panic = 'abort'     # Abort on panic
+strip = true        # Strip symbols from binary*

MANIFEST.in

@@ -6,3 +6,4 @@ global-include py.typed
 recursive-include scripts *.py
 recursive-include tests *.py
 recursive-include src *.rs
+include tiktoken *.json

core/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "tiktoken_core"
+version = "0.2.0"
+edition = "2021"
+rust-version = "1.57.0"
+
+[lib]
+name = "_tiktoken_core"
+crate-type = ["lib"]
+
+[dependencies]
+# tiktoken dependencies
+fancy-regex = "0.10.0"
+regex = "1.7.0"
+rustc-hash = "1.1.0"
+bstr = "1.0.1"
+reqwest = { version = "0.11.14", features = ["blocking"] }
+sha1 = "0.10.5"
+json = "0.12.4"
+base64 = "0.21.0"
+lazy_static = "1.4.0"