diff --git a/.github/workflows/consistency-ci.yml b/.github/workflows/consistency-ci.yml index 1cffaa2f9..9f9c310bc 100644 --- a/.github/workflows/consistency-ci.yml +++ b/.github/workflows/consistency-ci.yml @@ -84,7 +84,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -93,7 +93,7 @@ jobs: with: use-cross: true command: build - args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features' + args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release' - name: Build with Maven run: | mkdir -p rust/target/release diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 4e96d9e2a..d0ded9c9f 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -24,7 +24,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -37,7 +37,7 @@ jobs: with: use-cross: true command: build - args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features' + args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --features hdfs' - uses: actions/upload-artifact@v4 with: name: lakesoul-nativeio-x86_64-unknown-linux-gnu @@ -65,7 +65,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -73,7 +73,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: build - args: '--manifest-path rust/Cargo.toml --release --all-features' + args: '--manifest-path rust/Cargo.toml --release --features hdfs' - uses: actions/upload-artifact@v4 with: name: lakesoul-nativeio-x86_64-pc-windows-msvc @@ -103,7 +103,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -111,7 +111,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: build - args: '--manifest-path rust/Cargo.toml --release --all-features' + args: '--manifest-path rust/Cargo.toml --release --features hdfs' - uses: actions/upload-artifact@v4 with: name: lakesoul-nativeio-x86_64-apple-darwin diff --git a/.github/workflows/flink-cdc-hdfs-test.yml b/.github/workflows/flink-cdc-hdfs-test.yml index c4f8288da..54c96feef 100644 --- a/.github/workflows/flink-cdc-hdfs-test.yml +++ b/.github/workflows/flink-cdc-hdfs-test.yml @@ -69,7 +69,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -85,7 +85,7 @@ jobs: with: use-cross: true command: build - args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features' + args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --features hdfs' - name: Build with Maven run: | mkdir -p rust/target/release diff --git a/.github/workflows/flink-cdc-test.yml b/.github/workflows/flink-cdc-test.yml index bc124102b..1b8eaef6e 100644 --- a/.github/workflows/flink-cdc-test.yml +++ b/.github/workflows/flink-cdc-test.yml @@ -71,7 +71,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -87,7 +87,7 @@ jobs: with: use-cross: true command: build - args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features' + args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release' - name: Build with Maven run: | mkdir -p rust/target/release diff --git a/.github/workflows/maven-test.yml b/.github/workflows/maven-test.yml index 97e308b75..b7d25ba34 100644 --- a/.github/workflows/maven-test.yml +++ b/.github/workflows/maven-test.yml @@ -43,7 +43,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -56,7 +56,7 @@ jobs: with: use-cross: true command: build - args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features' + args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --features hdfs' - uses: actions/upload-artifact@v4 with: name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index 94069db17..713ab771a 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -36,7 +36,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -50,7 +50,7 @@ jobs: with: use-cross: true command: build - args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features' + args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --features hdfs' - uses: actions/upload-artifact@v4 with: name: lakesoul-nativeio-x86_64-unknown-linux-gnu @@ -78,7 +78,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -87,7 +87,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: build - args: '--manifest-path rust/Cargo.toml --release --all-features' + args: '--manifest-path rust/Cargo.toml --release --features hdfs' - uses: actions/upload-artifact@v4 with: name: lakesoul-nativeio-x86_64-pc-windows-msvc @@ -117,7 +117,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -126,7 +126,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: build - args: '--manifest-path rust/Cargo.toml --release --all-features' + args: '--manifest-path rust/Cargo.toml --release --features hdfs' - uses: actions/upload-artifact@v4 with: name: lakesoul-nativeio-x86_64-apple-darwin diff --git a/.github/workflows/presto-cdc-test.yml b/.github/workflows/presto-cdc-test.yml index b894351a4..a68425ee5 100644 --- a/.github/workflows/presto-cdc-test.yml +++ b/.github/workflows/presto-cdc-test.yml @@ -71,7 +71,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable default: true - uses: Swatinem/rust-cache@v2 with: @@ -87,7 +87,7 @@ jobs: with: use-cross: true command: build - args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features' + args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release' - name: Build with Maven run: | mkdir -p rust/target/release diff --git a/.github/workflows/rust-clippy.yml b/.github/workflows/rust-clippy.yml index 5cbd82500..cdad5f1fa 100644 --- a/.github/workflows/rust-clippy.yml +++ b/.github/workflows/rust-clippy.yml @@ -30,7 +30,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2023-05-20 + toolchain: stable components: clippy default: true - name: Install Protoc @@ -42,5 +42,5 @@ jobs: workspaces: "./rust -> target" key: "ubuntu-latest-clippy" - name: Run Clippy - run: cd rust && cargo clippy --no-deps --all-features --package lakesoul-io-c --package lakesoul-metadata-c + run: cd rust && cargo clippy --no-deps --features hdfs --package lakesoul-io-c --package lakesoul-metadata-c diff --git a/native-io/lakesoul-io-java/src/main/java/org/apache/arrow/c/jni/JniLoader.java b/native-io/lakesoul-io-java/src/main/java/org/apache/arrow/c/jni/JniLoader.java deleted file mode 100644 index 692dfff55..000000000 --- a/native-io/lakesoul-io-java/src/main/java/org/apache/arrow/c/jni/JniLoader.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This file is modified from https://github.com/apache/arrow/blob/apache-arrow-11.0.0/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java - * to fix loading dynamic libraries on Windows platform. - */ - -package org.apache.arrow.c.jni; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Set; - -/** - * The JniLoader for C Data Interface API's native implementation. - */ -public class JniLoader { - private static final JniLoader INSTANCE = new JniLoader(Collections.singletonList("arrow_cdata_jni")); - - public static JniLoader get() { - return INSTANCE; - } - - private final Set librariesToLoad; - - private JniLoader(List libraryNames) { - librariesToLoad = new HashSet<>(libraryNames); - } - - private boolean finished() { - return librariesToLoad.isEmpty(); - } - - /** - * If required JNI libraries are not loaded, then load them. - */ - public void ensureLoaded() { - if (finished()) { - return; - } - loadRemaining(); - } - - private synchronized void loadRemaining() { - // The method is protected by a mutex via synchronized, if more than one thread - // race to call - // loadRemaining, at same time only one will do the actual loading and the - // others will wait for - // the mutex to be acquired then check on the remaining list: if there are - // libraries that were not - // successfully loaded then the mutex owner will try to load them again. - if (finished()) { - return; - } - List libs = new ArrayList<>(librariesToLoad); - for (String lib : libs) { - load(lib); - librariesToLoad.remove(lib); - } - } - - private void load(String name) { - final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(name); - try { - File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir"))); - temp.deleteOnExit(); - try (final InputStream is = JniWrapper.class.getClassLoader().getResourceAsStream(libraryToLoad)) { - if (is == null) { - throw new FileNotFoundException(libraryToLoad); - } - Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); - System.load(temp.getAbsolutePath()); - } - } catch (IOException e) { - throw new IllegalStateException("error loading native libraries: " + e); - } - } - - private String getNormalizedArch() { - String arch = System.getProperty("os.arch").toLowerCase(Locale.US); - switch (arch) { - case "amd64": - arch = "x86_64"; - break; - case "aarch64": - arch = "aarch_64"; - break; - default: - break; - } - return arch; - } -} diff --git a/rust/Cross.toml b/rust/Cross.toml index 302642016..84cffd5ea 100644 --- a/rust/Cross.toml +++ b/rust/Cross.toml @@ -6,5 +6,4 @@ image = "dmetasoul/lakesoul-cross:v1.0.0" [target.aarch64-unknown-linux-gnu] -image.name = "dmetasoul/lakesoul-cross:v1.0.0-aarch64" -image.toolchain = ["aarch64-unknown-linux-gnu"] +image = "dmetasoul/lakesoul-cross:v1.0.0-aarch64" \ No newline at end of file diff --git a/rust/lakesoul-io-c/lakesoul_c_bindings.h b/rust/lakesoul-io-c/lakesoul_c_bindings.h index 043dd2f96..0241882a0 100644 --- a/rust/lakesoul-io-c/lakesoul_c_bindings.h +++ b/rust/lakesoul-io-c/lakesoul_c_bindings.h @@ -19,6 +19,10 @@ struct IOConfigBuilder { uint8_t private_[0]; }; +using c_ptrdiff_t = intptr_t; + +using c_size_t = uintptr_t; + struct IOConfig { uint8_t private_[0]; }; diff --git a/rust/lakesoul-io-c/src/lib.rs b/rust/lakesoul-io-c/src/lib.rs index 562f3b5a5..1e663fb78 100644 --- a/rust/lakesoul-io-c/src/lib.rs +++ b/rust/lakesoul-io-c/src/lib.rs @@ -3,10 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 #![allow(clippy::not_unsafe_ptr_arg_deref)] -#![feature(c_size_t)] extern crate core; -use core::ffi::{c_ptrdiff_t, c_size_t}; use std::ffi::{c_char, c_int, c_uchar, c_void, CStr, CString}; use std::io::Write; use std::ptr::NonNull; @@ -29,6 +27,9 @@ use lakesoul_io::lakesoul_reader::{LakeSoulReader, RecordBatch, Result, SyncSend use lakesoul_io::lakesoul_writer::SyncSendableMutableLakeSoulWriter; use log::debug; +pub type c_size_t = usize; +pub type c_ptrdiff_t = isize; + #[repr(C)] pub struct CResult { ptr: *mut OpaqueT, diff --git a/rust/lakesoul-metadata-c/src/lib.rs b/rust/lakesoul-metadata-c/src/lib.rs index a748c999c..6811896e7 100644 --- a/rust/lakesoul-metadata-c/src/lib.rs +++ b/rust/lakesoul-metadata-c/src/lib.rs @@ -2,11 +2,9 @@ // // SPDX-License-Identifier: Apache-2.0 -#![feature(c_size_t)] #![allow(clippy::not_unsafe_ptr_arg_deref)] extern crate core; -use core::ffi::c_ptrdiff_t; use std::collections::HashMap; use std::ffi::{c_char, c_uchar, CStr, CString}; use std::io::Write; @@ -21,6 +19,8 @@ use lakesoul_metadata::error::LakeSoulMetaDataError; use lakesoul_metadata::transfusion::SplitDesc; use proto::proto::entity; +pub type c_ptrdiff_t = isize; + #[repr(C)] pub struct CResult { ptr: *mut OpaqueT,