From 5f1a173562bbe1075d62bcf26734795274433507 Mon Sep 17 00:00:00 2001 From: Mimoja Date: Mon, 24 Jun 2024 19:13:01 +0200 Subject: [PATCH 01/11] clippy,nits: remove needless lifetimes clippy complains about these lifetimes as they can be removed by relying on lifetime elision instead. Signed-off-by: Mimoja --- src/dict_management.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dict_management.rs b/src/dict_management.rs index e9fdb37..e995fa5 100644 --- a/src/dict_management.rs +++ b/src/dict_management.rs @@ -7,8 +7,8 @@ use zstd::dict::{DecoderDictionary, EncoderDictionary}; // TODO: the rust interface currently requires a level when preparing a dictionary, but the zstd interface (ZSTD_CCtx_loadDictionary) does not. // TODO: Using LruCache here isn't very smart -pub fn encoder_dict_from_ctx<'a>( - ctx: &'a Context, +pub fn encoder_dict_from_ctx( + ctx: &Context, arg_index: usize, level: i32, ) -> anyhow::Result>> { @@ -48,8 +48,8 @@ pub fn encoder_dict_from_ctx<'a>( Ok(res) } -pub fn decoder_dict_from_ctx<'a>( - ctx: &'a Context, +pub fn decoder_dict_from_ctx( + ctx: &Context, arg_index: usize, ) -> anyhow::Result>> { use lru_time_cache::LruCache; From 634d08a4e6e02273e2b86b380183ffb35ed78ccd Mon Sep 17 00:00:00 2001 From: Mimoja Date: Mon, 24 Jun 2024 19:14:44 +0200 Subject: [PATCH 02/11] clippy,nits: remove needless borrow The borrow will be immediately dereferenced by the compiler and is therefore unnecessary. Signed-off-by: Mimoja --- src/add_functions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/add_functions.rs b/src/add_functions.rs index e91da1c..67332d0 100644 --- a/src/add_functions.rs +++ b/src/add_functions.rs @@ -148,7 +148,7 @@ pub mod tests { ]; let mut rng = rand::rngs::StdRng::seed_from_u64(seed); - let event_type_dist = WeightedIndex::new(&[10, 10, 1])?; + let event_type_dist = WeightedIndex::new([10, 10, 1])?; let window_properties_dist = WeightedIndex::new(window_properties.iter().map(|e| e.0))?; let app_id_dist = rand::distributions::Uniform::from(0..100); let data = (0..eles).map(|_| match event_type_dist.sample(&mut rng) { From d35ef5ca7868fe32605ea83fb6a6761b67a54450 Mon Sep 17 00:00:00 2001 From: Mimoja Date: Mon, 24 Jun 2024 19:15:44 +0200 Subject: [PATCH 03/11] clippy,nits: allow use of println! in tests Signed-off-by: Mimoja --- clippy.toml | 1 + 1 file changed, 1 insertion(+) create mode 100644 clippy.toml diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..40b1dda --- /dev/null +++ b/clippy.toml @@ -0,0 +1 @@ +allow-print-in-tests = true From 12fec03d275164fe97fd1b8e465785da04c06e07 Mon Sep 17 00:00:00 2001 From: Mimoja Date: Mon, 24 Jun 2024 22:51:38 +0200 Subject: [PATCH 04/11] clippy,nits: fix more style nits Signed-off-by: Mimoja --- src/add_functions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/add_functions.rs b/src/add_functions.rs index 67332d0..ecd8b1e 100644 --- a/src/add_functions.rs +++ b/src/add_functions.rs @@ -58,7 +58,7 @@ pub mod tests { use super::*; use anyhow::Context; use chrono::TimeZone; - pub use pretty_assertions::{assert_eq, assert_ne}; + pub use pretty_assertions::assert_eq; use rusqlite::{params, Connection}; use serde::{Deserialize, Serialize}; From b81df7710e971dc6f8b60cbe450e235b0f44c3b7 Mon Sep 17 00:00:00 2001 From: Mimoja Date: Sat, 6 Jul 2024 12:48:38 +0200 Subject: [PATCH 05/11] nits: fix even more clippy errors Signed-off-by: Mimoja --- src/add_functions.rs | 2 +- src/bin/benchmark.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/add_functions.rs b/src/add_functions.rs index ecd8b1e..530badd 100644 --- a/src/add_functions.rs +++ b/src/add_functions.rs @@ -198,7 +198,7 @@ pub mod tests { } fn test_strings() -> anyhow::Result> { - let data = vec![ + let data = [ "hello this is a test", "foobar", "looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong", diff --git a/src/bin/benchmark.rs b/src/bin/benchmark.rs index ba4a599..7133453 100644 --- a/src/bin/benchmark.rs +++ b/src/bin/benchmark.rs @@ -76,13 +76,13 @@ impl Bench for SelectBench { } fn execute(&self, conn: &Connection) -> Result { let mut stmt = conn.prepare("select data from title_basics where id = ?")?; - let mut total_len = 0; + let mut _total_len = 0; for id in &self.ids { let data: String = stmt.query_row(params![id], |r| r.get(0))?; - total_len += data.len(); + _total_len += data.len(); } - // eprintln!("total bytes got: {}", total_len); + // eprintln!("total bytes got: {}", _total_len); Ok(self.ids.len() as i64) } } @@ -213,7 +213,7 @@ fn main() -> Result<()> { .map(|preparer| { eprintln!("running preparer {its_per_bench} times"); (0..its_per_bench) - .map(|i| preparer(&db1)) + .map(|_i| preparer(&db1)) .collect::>() .context("preparing benches") }) @@ -237,7 +237,7 @@ fn main() -> Result<()> { let db_path = Path::new(&location).join(file_name); if !db_path.exists() { eprintln!("copying {} -> {}", input_db, db_path.to_string_lossy()); - std::fs::copy(&input_db, &db_path)?; + std::fs::copy(input_db, &db_path)?; } else { eprintln!( "{} already exists, assuming it's the same", From 27bdbd12a46270524f3cdd9034b9145270b2a624 Mon Sep 17 00:00:00 2001 From: Mimoja Date: Sat, 6 Jul 2024 12:58:13 +0200 Subject: [PATCH 06/11] test updated workflow for aarch64 Signed-off-by: Mimoja --- .github/workflows/pypi_release.yml | 49 ++++++++++++++++++------------ 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pypi_release.yml b/.github/workflows/pypi_release.yml index d46e02b..5ab9620 100644 --- a/.github/workflows/pypi_release.yml +++ b/.github/workflows/pypi_release.yml @@ -1,7 +1,7 @@ name: Publish to PyPI on: - push: + workflow_dispatch: jobs: build_wheels: @@ -9,7 +9,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-11] + os: [ubuntu-latest] + #cibw_arch: ["x86_64", "aarch64"] + #cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] + cibw_arch: ["aarch64"] + cibw_python: ["cp311-*", "cp312-*"] + env: CIBW_BEFORE_ALL_LINUX: "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y" CIBW_BUILD_VERBOSITY: "1" @@ -20,13 +25,18 @@ jobs: - uses: actions/checkout@v4 - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v1 + if: matrix.os == 'ubuntu-latest' && matrix.cibw_arch == 'aarch64' + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 with: - platforms: all + platforms: arm64 - name: Build wheels uses: pypa/cibuildwheel@v2.16.2 + env: + CIBW_BUILD_VERBOSITY: 1 + CIBW_BUILD: ${{ matrix.cibw_python }} + CIBW_ARCHS: ${{ matrix.cibw_arch }} + CIBW_TEST_SKIP: "*universal2:arm64" with: package-dir: ./python output-dir: ./python/wheelhouse @@ -51,7 +61,7 @@ jobs: - uses: actions/setup-python@v2 name: Install Python with: - python-version: "3.9" + python-version: "3.12" - name: Build sdist run: | @@ -62,19 +72,18 @@ jobs: - uses: actions/upload-artifact@v2 with: name: sdist-${{ runner.os }} - path: python/dist/*.tar.gz + path: dist/*.tar.* -# release: -# needs: [build_wheels, build_sdist] -# runs-on: ubuntu-latest -# steps: -# - uses: actions/download-artifact@v2 -# with: -# name: artifact -# path: python/dist + release: + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v2 + with: + name: dist + path: dist/ -# - uses: pypa/gh-action-pypi-publish@release/v1 -# with: -# packages-dir: python/dist/ -# user: __token__ -# password: ${{ secrets.PYPI_API_TOKEN }} + - uses: pypa/gh-action-pypi-publish@v1.8.10 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From 6a9f3cd81a1487b2dc9afceb9e92e2ed57d867e2 Mon Sep 17 00:00:00 2001 From: Mimoja Date: Sat, 6 Jul 2024 13:12:55 +0200 Subject: [PATCH 07/11] pypi: Build everything Signed-off-by: Mimoja --- .github/workflows/pypi_release.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pypi_release.yml b/.github/workflows/pypi_release.yml index 5ab9620..0e6da88 100644 --- a/.github/workflows/pypi_release.yml +++ b/.github/workflows/pypi_release.yml @@ -5,16 +5,13 @@ on: jobs: build_wheels: - name: Build wheels on ${{ matrix.os }} + name: Build wheels on ${{ matrix.os }} for ${{ matrix.cibw_python }} on ${{ matrix.cibw_arch }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] - #cibw_arch: ["x86_64", "aarch64"] - #cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] - cibw_arch: ["aarch64"] - cibw_python: ["cp311-*", "cp312-*"] - + cibw_arch: ["x86_64", "aarch64"] + cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] env: CIBW_BEFORE_ALL_LINUX: "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y" CIBW_BUILD_VERBOSITY: "1" From 4c191d451caea1cba7b016f4d01c7eb9de3657cb Mon Sep 17 00:00:00 2001 From: Mimoja Date: Sat, 6 Jul 2024 13:18:32 +0200 Subject: [PATCH 08/11] ci/cd: only test if pushed to main or master Signed-off-by: Mimoja --- .github/workflows/cicd.yml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 27f3e44..1e6013a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -4,11 +4,12 @@ name: CICD env: PROJECT_NAME: sqlite_zstd - #PROJECT_DESC: "sqlite ♥ zstd" - #PROJECT_AUTH: "bootandy" - #RUST_MIN_SRV: "1.31.0" - -on: [push, pull_request] +on: + push: + branches: + - main + - master + pull_request: jobs: style: @@ -65,21 +66,15 @@ jobs: # { os, target, cargo-options, features, use-cross, toolchain } - { os: ubuntu-latest, - target: arm-unknown-linux-gnueabihf, - use-cross: use-cross, + target: x86_64-unknown-linux-gnu, features: build_extension, } - { - os: ubuntu-18.04, - target: x86_64-unknown-linux-gnu, + os: ubuntu-latest, + target: arm-unknown-linux-gnueabihf, use-cross: use-cross, features: build_extension, } - #- { - # os: ubuntu-18.04, - # target: x86_64-unknown-linux-musl, - # use-cross: use-cross, - # } - { os: macos-latest, target: x86_64-apple-darwin, From 9fb12a62b4e3f1444c3277aaaa951f4921e40bdb Mon Sep 17 00:00:00 2001 From: Mimoja Date: Sat, 6 Jul 2024 13:30:46 +0200 Subject: [PATCH 09/11] dont build for py3.8 Signed-off-by: Mimoja --- .github/workflows/cicd.yml | 2 +- .github/workflows/pypi_release.yml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 1e6013a..4125945 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -166,7 +166,7 @@ jobs: echo set-output name=CARGO_USE_CROSS::${CARGO_USE_CROSS:-/false} echo ::set-output name=CARGO_USE_CROSS::${CARGO_USE_CROSS} # * strip executable? - STRIP="strip" ; case ${{ matrix.job.target }} in arm-unknown-linux-gnueabihf) STRIP="arm-linux-gnueabihf-strip" ;; *-pc-windows-msvc) STRIP="" ;; esac; + STRIP="strip" ; case ${{ matrix.job.target }} in arm-unknown-linux-gnueabihf) STRIP="arm-linux-gnueabihf-strip" ;; *-pc-windows-msvc) STRIP="" ;; *-apple-darwin) STRIP="strip -sX" ;; esac; echo set-output name=STRIP::${STRIP} echo ::set-output name=STRIP::${STRIP} - name: Create all needed build/work directories diff --git a/.github/workflows/pypi_release.yml b/.github/workflows/pypi_release.yml index 0e6da88..b81af18 100644 --- a/.github/workflows/pypi_release.yml +++ b/.github/workflows/pypi_release.yml @@ -11,11 +11,9 @@ jobs: matrix: os: [ubuntu-latest] cibw_arch: ["x86_64", "aarch64"] - cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] env: CIBW_BEFORE_ALL_LINUX: "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y" CIBW_BUILD_VERBOSITY: "1" - CIBW_SKIP: cp39-musllinux_i686 cp310-musllinux_i686 cp311-musllinux_i686 cp312-musllinux_i686 # Can't install Rust on musl based Linux systems CIBW_ENVIRONMENT: 'PATH="$PATH:$HOME/.cargo/bin"' steps: @@ -40,7 +38,7 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: wheel-${{ runner.os }} + name: dist path: ./python/wheelhouse/*.whl build_sdist: @@ -68,19 +66,21 @@ jobs: - uses: actions/upload-artifact@v2 with: - name: sdist-${{ runner.os }} - path: dist/*.tar.* + name: dist + path: python/dist/*.tar.* release: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: dist - path: dist/ + path: python/dist/ - uses: pypa/gh-action-pypi-publish@v1.8.10 with: + repository-url: https://pypi.org/project/sqlite-zstd-build user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + packages-dir: python/dist/* From 2c0707c5e9c433971f0a05333ef0c2182712a832 Mon Sep 17 00:00:00 2001 From: Mimoja Date: Sat, 6 Jul 2024 15:54:15 +0200 Subject: [PATCH 10/11] set strip params for osx Signed-off-by: Mimoja --- .github/workflows/cicd.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 4125945..7aa2ab7 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -166,9 +166,11 @@ jobs: echo set-output name=CARGO_USE_CROSS::${CARGO_USE_CROSS:-/false} echo ::set-output name=CARGO_USE_CROSS::${CARGO_USE_CROSS} # * strip executable? - STRIP="strip" ; case ${{ matrix.job.target }} in arm-unknown-linux-gnueabihf) STRIP="arm-linux-gnueabihf-strip" ;; *-pc-windows-msvc) STRIP="" ;; *-apple-darwin) STRIP="strip -sX" ;; esac; + STRIP="strip" ; STRIP_PARAMS="" ; case ${{ matrix.job.target }} in arm-unknown-linux-gnueabihf) STRIP="arm-linux-gnueabihf-strip" ;; *-pc-windows-msvc) STRIP="" ;; *-apple-darwin) STRIP_PARAMS="-x" ;; esac; echo set-output name=STRIP::${STRIP} echo ::set-output name=STRIP::${STRIP} + echo set-output name=STRIP_PARAMS::${STRIP_PARAMS} + echo ::set-output name=STRIP_PARAMS::${STRIP_PARAMS} - name: Create all needed build/work directories shell: bash run: | @@ -207,7 +209,7 @@ jobs: # binary cp 'target/${{ matrix.job.target }}/release/${{ steps.vars.outputs.LIB_FNAME }}' '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/' # `strip` binary (if needed) - if [ -n "${{ steps.vars.outputs.STRIP }}" ]; then "${{ steps.vars.outputs.STRIP }}" '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/${{ steps.vars.outputs.LIB_FNAME }}' ; fi + if [ -n "${{ steps.vars.outputs.STRIP }}" ]; then "${{ steps.vars.outputs.STRIP }}" ${{ steps.vars.outputs.STRIP_PARAMS }} '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/${{ steps.vars.outputs.LIB_FNAME }}' ; fi # README and LICENSE cp README.md '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/' cp LICENSE '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/' From 874471cd584dd77fa07d50ecbae10bc4046f326c Mon Sep 17 00:00:00 2001 From: Mimoja Date: Sat, 6 Jul 2024 17:46:41 +0200 Subject: [PATCH 11/11] fix pypi --- .github/workflows/pypi_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_release.yml b/.github/workflows/pypi_release.yml index b81af18..9176272 100644 --- a/.github/workflows/pypi_release.yml +++ b/.github/workflows/pypi_release.yml @@ -83,4 +83,4 @@ jobs: repository-url: https://pypi.org/project/sqlite-zstd-build user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - packages-dir: python/dist/* + packages-dir: python/dist