From 439456d6d9bd0a7148f5cb495e2caf134483d528 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 01:47:34 +0800 Subject: [PATCH 01/38] fix(ci): enforce the Java 17 build contract - reject unsupported candidate SDK build JDKs early - validate exact Java 17 provenance in distributions - isolate the pinned legacy Server compatibility JVM --- .github/scripts/test_candidate_sdk.py | 13 +++++++++++++ .github/scripts/verify_candidate_image_sdk.py | 2 +- .github/workflows/client-go-ci.yml | 12 ++++++++---- docs/java17-migration.md | 2 ++ .../assembly/travis/install-candidate-sdk.sh | 6 +++--- .../assembly/travis/install-server.test.cjs | 10 +++++++++- 6 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.github/scripts/test_candidate_sdk.py b/.github/scripts/test_candidate_sdk.py index ef152ffa4..b56b580fc 100644 --- a/.github/scripts/test_candidate_sdk.py +++ b/.github/scripts/test_candidate_sdk.py @@ -183,6 +183,19 @@ def test_reactor_installs_do_not_change_the_locked_sdk_contract(self): with self.assertRaisesRegex(RuntimeError, "SDK artifact hash mismatch"): sdk.validate_distribution(repository, directory, "hubble") + def test_candidate_manifest_requires_the_supported_build_jdk(self): + with tempfile.TemporaryDirectory() as temporary: + repository, manifest = self.fixture(Path(temporary)) + for version in ("11", "18", "21"): + with self.subTest(version=version): + manifest["java_version"] = version + self.write_manifest(repository, manifest) + with self.assertRaisesRegex(RuntimeError, "Java version"): + sdk.validate_sdk(repository) + manifest["java_version"] = "17" + self.write_manifest(repository, manifest) + sdk.validate_sdk(repository) + def test_malformed_manifests_have_controlled_diagnostics(self): with tempfile.TemporaryDirectory() as temporary: repository, manifest = self.fixture(Path(temporary)) diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index 19f10de8e..8c95fdc7d 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -64,7 +64,7 @@ def _validate_sdk(repository): manifest = json.loads((repository / "candidate-sdk-manifest.json").read_text()) if (manifest["repository"], manifest["commit"]) != (REPOSITORY, COMMIT): raise RuntimeError("SDK source does not match the locked candidate") - if manifest["source_revision"] != "1.7.0" or int(manifest["java_version"]) < 17: + if manifest["source_revision"] != "1.7.0" or int(manifest["java_version"]) != 17: raise RuntimeError("SDK revision or Java version does not match the candidate contract") modules = manifest["required_sdk_modules"] if len(modules) != len(REQUIRED_MODULES) or {m["source_pom"] for m in modules} != REQUIRED_MODULES: diff --git a/.github/workflows/client-go-ci.yml b/.github/workflows/client-go-ci.yml index 87c6663f5..a85da2062 100644 --- a/.github/workflows/client-go-ci.yml +++ b/.github/workflows/client-go-ci.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - JAVA_VERSION: ['11'] + SERVER_JAVA_VERSION: ['11'] steps: - name: Fetch Code @@ -39,14 +39,18 @@ jobs: with: fetch-depth: 2 - - name: Get HugeGraph stable commit id + - name: Resolve the released Server 1.7 fixture id: get-commit uses: ./.github/actions/get-hugegraph-commit + with: + repository: apache/hugegraph + ref: refs/tags/1.7.0 + expected-commit: b12425c2032bf0d21a97b8221f42a18055c2982f - - name: Setup Java environment + - name: Set up Java 11 for the released Server fixture uses: ./.github/actions/setup-java-env with: - java-version: ${{ matrix.JAVA_VERSION }} + java-version: ${{ matrix.SERVER_JAVA_VERSION }} distribution: 'zulu' use-stage: ${{ env.USE_STAGE }} diff --git a/docs/java17-migration.md b/docs/java17-migration.md index 33e534595..2daa2a818 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -2,6 +2,8 @@ Toolchain 1.8 requires Java 17 for the Java Client, ordinary Loader, Tools and Hubble. Select Java 17 for applications and for every Spark driver and executor using the new Client. An older server can keep its own supported JVM. +Toolchain builds, tests and applications use Java 17. The released Server 1.7 compatibility fixture retains its supported Java 11 in a separate process; its JVM must not select the Toolchain build or test JVM. Building the candidate Server SDK requires JDK 17, matching the server reactor's `[17,18)` Enforcer range. + ## Build the locked candidate The Java 17 server candidate retains the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. The locked source is `hugegraph/hugegraph`, branch `codex/cypher-minimal-compat`, commit `4c162f539b906fa06dd83228e3691b77fa5b77d7`. diff --git a/hugegraph-client/assembly/travis/install-candidate-sdk.sh b/hugegraph-client/assembly/travis/install-candidate-sdk.sh index 58a6e2936..abf798af7 100755 --- a/hugegraph-client/assembly/travis/install-candidate-sdk.sh +++ b/hugegraph-client/assembly/travis/install-candidate-sdk.sh @@ -23,13 +23,13 @@ if [[ $# -ne 3 || "$2" != /* || -e "$2" || "$3" != /* || ! -d "$3" || -e "$3/org fi if [[ -z "${JAVA_HOME:-}" || ! -x "$JAVA_HOME/bin/java" ]]; then - echo "Set JAVA_HOME to an installed JDK 17 or newer before building the candidate SDK" >&2 + echo "Set JAVA_HOME to an installed JDK 17 before building the candidate SDK" >&2 exit 1 fi JAVA_VERSION=$("$JAVA_HOME/bin/java" -XshowSettings:properties -version 2>&1 | awk -F ' = ' '/java.specification.version =/ {print $2}') -if [[ ! "$JAVA_VERSION" =~ ^[0-9]+$ || "$JAVA_VERSION" -lt 17 ]]; then - echo "Candidate SDK requires JDK 17 or newer, found: $JAVA_VERSION" >&2 +if [[ ! "$JAVA_VERSION" =~ ^[0-9]+$ || "$JAVA_VERSION" -ne 17 ]]; then + echo "Candidate SDK requires JDK 17, found: $JAVA_VERSION" >&2 exit 1 fi export PATH="$JAVA_HOME/bin:$PATH" diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs index 28cbf565e..5977a6150 100644 --- a/hugegraph-client/assembly/travis/install-server.test.cjs +++ b/hugegraph-client/assembly/travis/install-server.test.cjs @@ -428,10 +428,18 @@ test('candidate SDK rejects a moved source before Maven or manifest publication' assert.equal(existsSync(join(repo, 'candidate-sdk-manifest.json')), false); }); -test('candidate SDK refuses missing or legacy JVMs and existing HugeGraph artifact repositories', t => { +test('candidate SDK refuses missing JVMs, unsupported build JVMs and existing artifact repositories', t => { const { root, repo, env, run } = candidateSdkFixture(t); assert.notEqual(run('missing-jvm', undefined, { JAVA_HOME: '' }).status, 0); assert.notEqual(run('legacy-jvm', undefined, { SDK_JAVA_VERSION: '11' }).status, 0); + for (const version of ['18', '21']) { + const result = run(`unsupported-jvm-${version}`, undefined, { SDK_JAVA_VERSION: version }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires JDK 17/); + assert.equal(existsSync(join(root, `unsupported-jvm-${version}`)), false); + } + assert.equal(existsSync(env.BUILD_ARGS), false); + assert.equal(existsSync(join(repo, 'candidate-sdk-manifest.json')), false); mkdirSync(join(repo, 'org/apache/hugegraph'), { recursive: true }); const result = run('existing-repo'); assert.notEqual(result.status, 0); From a1ed2735bec844ea49a62bc7cf89cb09b8c04a02 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 01:52:08 +0800 Subject: [PATCH 02/38] fix(ci): validate the declared JDK without coercion - require the SDK generator Java version format - reject numeric and fractional manifest versions - preserve controlled malformed manifest diagnostics --- .github/scripts/test_candidate_sdk.py | 6 ++++++ .github/scripts/verify_candidate_image_sdk.py | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/scripts/test_candidate_sdk.py b/.github/scripts/test_candidate_sdk.py index b56b580fc..5cbb88290 100644 --- a/.github/scripts/test_candidate_sdk.py +++ b/.github/scripts/test_candidate_sdk.py @@ -192,6 +192,12 @@ def test_candidate_manifest_requires_the_supported_build_jdk(self): self.write_manifest(repository, manifest) with self.assertRaisesRegex(RuntimeError, "Java version"): sdk.validate_sdk(repository) + for version in (17.9, 17, True, "17.0"): + with self.subTest(version=version): + manifest["java_version"] = version + self.write_manifest(repository, manifest) + with self.assertRaisesRegex(RuntimeError, "Invalid candidate SDK manifest"): + sdk.validate_sdk(repository) manifest["java_version"] = "17" self.write_manifest(repository, manifest) sdk.validate_sdk(repository) diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index 8c95fdc7d..e4331fa22 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -64,7 +64,10 @@ def _validate_sdk(repository): manifest = json.loads((repository / "candidate-sdk-manifest.json").read_text()) if (manifest["repository"], manifest["commit"]) != (REPOSITORY, COMMIT): raise RuntimeError("SDK source does not match the locked candidate") - if manifest["source_revision"] != "1.7.0" or int(manifest["java_version"]) != 17: + java_version = manifest["java_version"] + if not isinstance(java_version, str) or not java_version.isdecimal(): + raise ValueError("Invalid Java version format") + if manifest["source_revision"] != "1.7.0" or java_version != "17": raise RuntimeError("SDK revision or Java version does not match the candidate contract") modules = manifest["required_sdk_modules"] if len(modules) != len(REQUIRED_MODULES) or {m["source_pom"] for m in modules} != REQUIRED_MODULES: From 227380f9c149ee4c9cf9bef5820ef7444a053802 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 21:28:01 +0800 Subject: [PATCH 03/38] fix(ci): start HDFS from the pinned image - reuse the existing Hadoop 3.3.6 container fixture - remove the unused Hadoop archive cache - collect HDFS logs and remove containers after tests --- .github/workflows/loader-ci.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 24a8ba9d3..d0ec8173b 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -64,18 +64,12 @@ jobs: id: candidate-sdk uses: ./.github/actions/setup-candidate-sdk - - name: Cache Hadoop - uses: actions/cache@v6 - with: - path: ~/hadoop-3.3.6.tar.gz - key: ${{ runner.os }}-hadoop-3.3.6 - - name: Compile run: | mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp - name: Prepare Hadoop service - run: bash "$TRAVIS_DIR/install-hadoop.sh" + run: bash "$TRAVIS_DIR/start-hdfs-ci.sh" - name: Setup released Server fixture if: matrix.SERVER_BASELINE == 'release-1.7' @@ -110,3 +104,14 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: hugegraph-loader/target/jacoco.xml + + - name: Collect HDFS diagnostics and stop containers + if: always() + run: | + for service in namenode datanode; do + container="hugegraph-ci-hdfs-$service" + if docker container inspect "$container" > /dev/null 2>&1; then + docker logs "$container" || true + docker rm --force "$container" + fi + done From 0596e708a5579121137dd567957ea324e6ba9a22 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 21:31:31 +0800 Subject: [PATCH 04/38] fix(ci): flatten candidate SDK root model - install concrete CI-friendly parent coordinates - retain same-source reactor and isolated repository - preserve license metadata processing and verification --- hugegraph-client/assembly/travis/install-candidate-sdk.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hugegraph-client/assembly/travis/install-candidate-sdk.sh b/hugegraph-client/assembly/travis/install-candidate-sdk.sh index 276c2fab4..97271ed05 100755 --- a/hugegraph-client/assembly/travis/install-candidate-sdk.sh +++ b/hugegraph-client/assembly/travis/install-candidate-sdk.sh @@ -44,7 +44,11 @@ CANDIDATE_REPO=$3 bash "$SCRIPT_DIR/checkout-server.sh" "$1" "$SOURCE_DIR" # Install the entire locked reactor so Common, Core, PD and Store share one source. # The fixture installers retain package-only semantics and use another repository. -(cd "$SOURCE_DIR" && mvn "-Dmaven.repo.local=$CANDIDATE_REPO" install \ +# Flatten the root too: its installed ${revision} otherwise breaks dependency +# model reconstruction while remote-resources generates license metadata. +(cd "$SOURCE_DIR" && mvn "-Dmaven.repo.local=$CANDIDATE_REPO" \ + org.codehaus.mojo:flatten-maven-plugin:1.3.0:flatten install \ + -Dflatten.mode=resolveCiFriendliesOnly -DupdatePomFile=true \ -DskipTests -Dmaven.javadoc.skip=true -ntp) ACTUAL_COMMIT=$(git -C "$SOURCE_DIR" rev-parse HEAD) python3 - "$SOURCE_DIR" "$CANDIDATE_REPO" "${SERVER_REPOSITORY:-apache/hugegraph}" \ From a745a4246b7b4cd41581d881460400b09268372d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 21:34:05 +0800 Subject: [PATCH 05/38] fix(ci): authenticate coverage with OIDC - route reports using the GitHub repository identity - ignore stale repository upload tokens - limit OIDC permissions to coverage jobs --- .github/actions/upload-coverage/action.yml | 8 +++++--- .github/workflows/ci.yml | 12 ++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml index bf8ba8cd8..6d742ea54 100644 --- a/.github/actions/upload-coverage/action.yml +++ b/.github/actions/upload-coverage/action.yml @@ -3,8 +3,8 @@ description: 'Upload test coverage report to Codecov' inputs: token: - description: 'Codecov upload token' - required: true + description: 'Legacy compatibility input; OIDC authenticates uploads instead' + required: false file: description: 'Path to the coverage report file' required: false @@ -16,6 +16,8 @@ runs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v7 with: - token: ${{ inputs.token }} + use_oidc: true + disable_search: true + fail_ci_if_error: true slug: ${{ github.repository }} files: ${{ inputs.file }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57709a2b1..9564c4125 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,6 +188,10 @@ jobs: java-version: '17' client: + permissions: + contents: read + actions: read + id-token: write needs: [plan, fixture] if: ${{ needs.plan.outputs.client == 'true' }} uses: ./.github/workflows/client-ci.yml @@ -199,6 +203,10 @@ jobs: secrets: inherit loader: + permissions: + contents: read + actions: read + id-token: write needs: [plan, fixture] if: ${{ needs.plan.outputs.loader == 'true' }} uses: ./.github/workflows/loader-ci.yml @@ -232,6 +240,10 @@ jobs: secrets: inherit hubble: + permissions: + contents: read + actions: read + id-token: write needs: [plan, fixture, hubble-fixture] if: ${{ needs.plan.outputs.hubble == 'true' }} uses: ./.github/workflows/hubble-ci.yml From 7b2b896abc3bf0bb5cb322eb86b27ee2e3f083e2 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 21:34:28 +0800 Subject: [PATCH 06/38] fix(ci): align SDK bootstrap regression test - expect native flatten goal before SDK installation - retain CI-friendly version and installed POM checks - verify the complete server fixture Node test suite --- hugegraph-client/assembly/travis/install-server.test.cjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs index a41f6d5da..7280ea0ac 100644 --- a/hugegraph-client/assembly/travis/install-server.test.cjs +++ b/hugegraph-client/assembly/travis/install-server.test.cjs @@ -404,7 +404,9 @@ test('candidate SDK installs the verified reactor into an explicit repository an const result = run('checkout'); assert.equal(result.status, 0, result.stderr); assert.deepEqual(readFileSync(env.BUILD_ARGS, 'utf8').trim().split('\n'), - [`-Dmaven.repo.local=${repo}`, 'install', '-DskipTests', '-Dmaven.javadoc.skip=true', '-ntp']); + [`-Dmaven.repo.local=${repo}`, 'org.codehaus.mojo:flatten-maven-plugin:1.3.0:flatten', 'install', + '-Dflatten.mode=resolveCiFriendliesOnly', '-DupdatePomFile=true', + '-DskipTests', '-Dmaven.javadoc.skip=true', '-ntp']); const manifest = JSON.parse(readFileSync(join(repo, 'candidate-sdk-manifest.json'), 'utf8')); assert.equal(manifest.repository, 'example/server'); assert.equal(manifest.commit, sha); From 7f587e2010f48d2ae1e49076840a298d7a8c455d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 21:33:59 +0800 Subject: [PATCH 07/38] fix(ci): prepare Spark from the official image Pin the official Spark 3.5.8 Java 17 image digest. Extract its verified runtime with Docker instead of archives. Expose the runtime home and clean up the temporary container. --- .../actions/setup-spark-runtime/action.yml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/actions/setup-spark-runtime/action.yml diff --git a/.github/actions/setup-spark-runtime/action.yml b/.github/actions/setup-spark-runtime/action.yml new file mode 100644 index 000000000..cba316663 --- /dev/null +++ b/.github/actions/setup-spark-runtime/action.yml @@ -0,0 +1,24 @@ +name: Setup Spark runtime +description: Extract the pinned official Spark 3.5.8 runtime for Java 17 engine tests +outputs: + home: + description: Local Spark runtime directory + value: ${{ steps.runtime.outputs.home }} +runs: + using: composite + steps: + - name: Install Spark 3.5.8 from the official image + id: runtime + shell: bash + run: | + image='apache/spark:3.5.8-scala2.12-java17-ubuntu@sha256:9577096950d62dd59994f84cc3a0902fdae994a4bf238933f2fdca22053e5adb' + docker pull --platform linux/amd64 "$image" + container=$(docker create --platform linux/amd64 "$image" /bin/true) + trap 'docker rm -f "$container" >/dev/null' EXIT + spark_home="$RUNNER_TEMP/spark-3.5.8" + docker cp "$container:/opt/spark" "$spark_home" + grep -q '^Spark 3\.5\.8 ' "$spark_home/RELEASE" + test -x "$spark_home/bin/spark-submit" + # Keep the launcher and independent executor on the job's Java 17 runtime. + mkdir -p "$spark_home/conf" "$spark_home/work" + printf 'home=%s\n' "$spark_home" >> "$GITHUB_OUTPUT" From cd29f0a9761a2b7f7068936a15c5eaf08e6a4667 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 21:46:25 +0800 Subject: [PATCH 08/38] refactor(ci): remove single-use Spark action - keep runtime preparation in its consuming job - retain the shared HDFS container fixture - avoid an extra CI abstraction --- .../actions/setup-spark-runtime/action.yml | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 .github/actions/setup-spark-runtime/action.yml diff --git a/.github/actions/setup-spark-runtime/action.yml b/.github/actions/setup-spark-runtime/action.yml deleted file mode 100644 index cba316663..000000000 --- a/.github/actions/setup-spark-runtime/action.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Setup Spark runtime -description: Extract the pinned official Spark 3.5.8 runtime for Java 17 engine tests -outputs: - home: - description: Local Spark runtime directory - value: ${{ steps.runtime.outputs.home }} -runs: - using: composite - steps: - - name: Install Spark 3.5.8 from the official image - id: runtime - shell: bash - run: | - image='apache/spark:3.5.8-scala2.12-java17-ubuntu@sha256:9577096950d62dd59994f84cc3a0902fdae994a4bf238933f2fdca22053e5adb' - docker pull --platform linux/amd64 "$image" - container=$(docker create --platform linux/amd64 "$image" /bin/true) - trap 'docker rm -f "$container" >/dev/null' EXIT - spark_home="$RUNNER_TEMP/spark-3.5.8" - docker cp "$container:/opt/spark" "$spark_home" - grep -q '^Spark 3\.5\.8 ' "$spark_home/RELEASE" - test -x "$spark_home/bin/spark-submit" - # Keep the launcher and independent executor on the job's Java 17 runtime. - mkdir -p "$spark_home/conf" "$spark_home/work" - printf 'home=%s\n' "$spark_home" >> "$GITHUB_OUTPUT" From b0ecd25d041d603b38348a7f40ec1d2ba2368afd Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 21:49:24 +0800 Subject: [PATCH 09/38] fix(ci): scope candidate build to SDK modules - build SDK and server distribution dependency closure - avoid released Toolchain dependencies from server tests - retain all required SDK provenance and license checks --- hugegraph-client/assembly/travis/install-candidate-sdk.sh | 6 +++++- hugegraph-client/assembly/travis/install-server.test.cjs | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/hugegraph-client/assembly/travis/install-candidate-sdk.sh b/hugegraph-client/assembly/travis/install-candidate-sdk.sh index 97271ed05..605b013e8 100755 --- a/hugegraph-client/assembly/travis/install-candidate-sdk.sh +++ b/hugegraph-client/assembly/travis/install-candidate-sdk.sh @@ -42,11 +42,15 @@ SERVER_FETCH_REF=${SERVER_FETCH_REF:-$1} export SERVER_REPOSITORY SERVER_FETCH_REF CANDIDATE_REPO=$3 bash "$SCRIPT_DIR/checkout-server.sh" "$1" "$SOURCE_DIR" -# Install the entire locked reactor so Common, Core, PD and Store share one source. +# Install the SDK and Server distribution dependency closure from one source. +# Server cluster tests pull an unrelated released Toolchain/SDK back into this build. # The fixture installers retain package-only semantics and use another repository. +SDK_MODULES=hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-client +SDK_MODULES+=,hugegraph-store/hg-store-client,hugegraph-struct # Flatten the root too: its installed ${revision} otherwise breaks dependency # model reconstruction while remote-resources generates license metadata. (cd "$SOURCE_DIR" && mvn "-Dmaven.repo.local=$CANDIDATE_REPO" \ + -pl "$SDK_MODULES" -am \ org.codehaus.mojo:flatten-maven-plugin:1.3.0:flatten install \ -Dflatten.mode=resolveCiFriendliesOnly -DupdatePomFile=true \ -DskipTests -Dmaven.javadoc.skip=true -ntp) diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs index 7280ea0ac..7c3fad105 100644 --- a/hugegraph-client/assembly/travis/install-server.test.cjs +++ b/hugegraph-client/assembly/travis/install-server.test.cjs @@ -404,7 +404,9 @@ test('candidate SDK installs the verified reactor into an explicit repository an const result = run('checkout'); assert.equal(result.status, 0, result.stderr); assert.deepEqual(readFileSync(env.BUILD_ARGS, 'utf8').trim().split('\n'), - [`-Dmaven.repo.local=${repo}`, 'org.codehaus.mojo:flatten-maven-plugin:1.3.0:flatten', 'install', + [`-Dmaven.repo.local=${repo}`, '-pl', + 'hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-client,hugegraph-store/hg-store-client,hugegraph-struct', + '-am', 'org.codehaus.mojo:flatten-maven-plugin:1.3.0:flatten', 'install', '-Dflatten.mode=resolveCiFriendliesOnly', '-DupdatePomFile=true', '-DskipTests', '-Dmaven.javadoc.skip=true', '-ntp']); const manifest = JSON.parse(readFileSync(join(repo, 'candidate-sdk-manifest.json'), 'utf8')); From fa7ef8705a06a33a881c7efd3456f60f3d284891 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 22:32:32 +0800 Subject: [PATCH 10/38] fix(ci): align SDK with ASF master - use the merged ASF Java 17 and TinkerPop source - retain exact SDK provenance and installed artifact checks - remove temporary source references from CI and guides --- .github/actions/setup-candidate-sdk/action.yml | 9 ++++----- .github/scripts/verify_candidate_image_sdk.py | 5 ++--- .github/workflows/ci.yml | 7 +++---- docs/ci.md | 2 +- docs/java17-migration.md | 11 ++++------- .../assembly/travis/install-candidate-sdk.sh | 3 +-- 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index 23e66b95f..3908e9ead 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -1,17 +1,16 @@ name: 'Setup same-source candidate SDK' description: 'Build a locked Java 17 server reactor outside the workspace in an isolated Maven repository' -# TODO(apache/hugegraph#3263): temporary pre-merge source; after merge use apache/hugegraph and a verified ASF SHA. inputs: repository: description: 'GitHub owner/repository for the candidate SDK' - default: 'hugegraph/hugegraph' + default: 'apache/hugegraph' ref: description: 'Fetchable candidate source ref' - default: 'codex/cypher-minimal-compat' + default: 'master' expected-commit: description: 'Locked full candidate source commit' - default: '4c162f539b906fa06dd83228e3691b77fa5b77d7' + default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' outputs: repository: @@ -58,7 +57,7 @@ runs: server_build=$(mktemp -d "$RUNNER_TEMP/hugegraph-candidate-build.XXXXXX") bash "$GITHUB_WORKSPACE/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ "$SERVER_COMMIT" "$server_build/hugegraph" "$candidate_repo" - archives=("$server_build"/hugegraph/hugegraph-server/apache-hugegraph-*.tar.gz) + archives=("$server_build"/apache/hugegraph-server/apache-hugegraph-*.tar.gz) if [[ ${#archives[@]} -ne 1 || ! -f "${archives[0]}" ]]; then printf 'Expected one candidate server archive, found: %s\n' "${archives[*]}" >&2 exit 1 diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index 12ebaf908..783f881a0 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -22,9 +22,8 @@ import json from pathlib import Path -# TODO(apache/hugegraph#3263): temporary pre-merge source; after merge use apache/hugegraph and a verified ASF SHA. -REPOSITORY = "hugegraph/hugegraph" -COMMIT = "4c162f539b906fa06dd83228e3691b77fa5b77d7" +REPOSITORY = "apache/hugegraph" +COMMIT = "d9abcd4317fb36128e7e4d209139ec7f3a28cdfc" REQUIRED_MODULES = { "pom.xml", "hugegraph-commons/pom.xml", "hugegraph-server/pom.xml", "hugegraph-pd/pom.xml", "hugegraph-store/pom.xml", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9564c4125..3736c5019 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,10 +85,9 @@ jobs: if: steps.scope.outputs.needsFixture == 'true' uses: ./.github/actions/get-hugegraph-commit with: - # TODO(apache/hugegraph#3263): switch the temporary source to ASF after merge. - repository: hugegraph/hugegraph - ref: codex/cypher-minimal-compat - expected-commit: 4c162f539b906fa06dd83228e3691b77fa5b77d7 + repository: apache/hugegraph + ref: master + expected-commit: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc - name: Record external test inputs env: NEEDS_SERVER: ${{ steps.scope.outputs.needsFixture }} diff --git a/docs/ci.md b/docs/ci.md index 0b7fb14a8..2086dd157 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -47,7 +47,7 @@ module tests. They do not publish an image. Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement the existing affected-module selection without adding unrelated modules. -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked candidate Server runs on Java 17. TODO ([apache/hugegraph#3263](https://github.com/apache/hugegraph/pull/3263)): replace the temporary candidate source with verified ASF inputs after the Server migration merges, and revalidate the SDK/runtime matrix. +The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. The released Server package and Hubble's candidate fixture are each built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. diff --git a/docs/java17-migration.md b/docs/java17-migration.md index c5c4ea8f0..5b7f1df72 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -6,9 +6,7 @@ Toolchain builds, tests and applications use Java 17. The released Server 1.7 co ## Build the locked candidate -TODO ([apache/hugegraph#3263](https://github.com/apache/hugegraph/pull/3263)): the organization repository, branch and commit below are temporary pre-merge validation inputs. After the Java 17 Server change merges, replace them with `apache/hugegraph` and a verified ASF commit, update the SDK action/verifier, Docker defaults, CI and examples together, and rerun dependency and runtime validation. Do not retain the temporary source after the merge. Prefer distinct, published SDK coordinates when they become available. - -The Java 17 server candidate retains the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. The locked source is `hugegraph/hugegraph`, branch `codex/cypher-minimal-compat`, commit `4c162f539b906fa06dd83228e3691b77fa5b77d7`. +The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses `apache/hugegraph` master with a verified ASF commit. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. Use an explicit JDK directory and check both Java and Maven. On systems where Java 17 is not registered, a system JDK selector can return another installed version. Checking the generated class version alone does not identify the JVM that built or tested it. @@ -23,10 +21,9 @@ mvn -version toolchain_root="$PWD" candidate_dir=$(mktemp -d /tmp/hugegraph-java17.XXXXXX) mkdir "$candidate_dir/m2" -# TODO(apache/hugegraph#3263): temporary pre-merge source; after merge use apache/hugegraph and a verified ASF SHA. -export SERVER_REPOSITORY=hugegraph/hugegraph -export SERVER_FETCH_REF=codex/cypher-minimal-compat -server_commit=4c162f539b906fa06dd83228e3691b77fa5b77d7 +export SERVER_REPOSITORY=apache/hugegraph +export SERVER_FETCH_REF=master +server_commit=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc bash "$toolchain_root/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ "$server_commit" "$candidate_dir/server" "$candidate_dir/m2" diff --git a/hugegraph-client/assembly/travis/install-candidate-sdk.sh b/hugegraph-client/assembly/travis/install-candidate-sdk.sh index 605b013e8..fcc81d6b2 100755 --- a/hugegraph-client/assembly/travis/install-candidate-sdk.sh +++ b/hugegraph-client/assembly/travis/install-candidate-sdk.sh @@ -36,8 +36,7 @@ export PATH="$JAVA_HOME/bin:$PATH" SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) SOURCE_DIR=$2 -# TODO(apache/hugegraph#3263): temporary pre-merge source; after merge use apache/hugegraph and a verified ASF SHA. -SERVER_REPOSITORY=${SERVER_REPOSITORY:-hugegraph/hugegraph} +SERVER_REPOSITORY=${SERVER_REPOSITORY:-apache/hugegraph} SERVER_FETCH_REF=${SERVER_FETCH_REF:-$1} export SERVER_REPOSITORY SERVER_FETCH_REF CANDIDATE_REPO=$3 From 3adcff31e74dc5e4527d90abf4d06c3f49ed1cd9 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 22:34:55 +0800 Subject: [PATCH 11/38] fix(images): build SDK from ASF master - use the merged upstream Java 17 source for image builds - keep builders aligned with SDK provenance validation - remove temporary repository and branch references --- hugegraph-hubble/Dockerfile | 7 +++---- hugegraph-loader/Dockerfile | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/hugegraph-hubble/Dockerfile b/hugegraph-hubble/Dockerfile index 2e004f1af..6ed45b30f 100644 --- a/hugegraph-hubble/Dockerfile +++ b/hugegraph-hubble/Dockerfile @@ -36,13 +36,12 @@ COPY hugegraph-client/assembly/travis/install-candidate-sdk.sh \ COPY .github/scripts/verify_candidate_image_sdk.py /pkg/.github/scripts/ # Direct builds bootstrap the same locked SDK; CI can supply a verified builder. -# TODO(apache/hugegraph#3263): after merge replace this temporary source with apache/hugegraph and an ASF SHA. RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ - SERVER_REPOSITORY=hugegraph/hugegraph \ - SERVER_FETCH_REF=codex/cypher-minimal-compat \ + SERVER_REPOSITORY=apache/hugegraph \ + SERVER_FETCH_REF=master \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - 4c162f539b906fa06dd83228e3691b77fa5b77d7 /opt/candidate-source /opt/candidate-m2; \ + d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 diff --git a/hugegraph-loader/Dockerfile b/hugegraph-loader/Dockerfile index 6cc99be19..2c5ec396a 100644 --- a/hugegraph-loader/Dockerfile +++ b/hugegraph-loader/Dockerfile @@ -32,13 +32,12 @@ COPY hugegraph-client/assembly/travis/install-candidate-sdk.sh \ COPY .github/scripts/verify_candidate_image_sdk.py /pkg/.github/scripts/ # Direct builds bootstrap the same locked SDK; CI can supply a verified builder. -# TODO(apache/hugegraph#3263): after merge replace this temporary source with apache/hugegraph and an ASF SHA. RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ - SERVER_REPOSITORY=hugegraph/hugegraph \ - SERVER_FETCH_REF=codex/cypher-minimal-compat \ + SERVER_REPOSITORY=apache/hugegraph \ + SERVER_FETCH_REF=master \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - 4c162f539b906fa06dd83228e3691b77fa5b77d7 /opt/candidate-source /opt/candidate-m2; \ + d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 From 800c59cd5b9f72304280aaf47ba07b3c362937e6 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 22:34:26 +0800 Subject: [PATCH 12/38] chore: remove obsolete Server source references - use ASF links in client documentation and provenance - align the Hubble archive fixture with the ASF repository - remove the unused legacy organization download script --- hugegraph-client/README.md | 2 +- .../org/apache/hugegraph/util/IdUtil.java | 2 +- .../hugegraph/util/SplicingIdGenerator.java | 2 +- .../travis/test_configure_gremlin_fixture.py | 3 +- .../travis/install-hugegraph-from-tar.sh | 49 ------------------- 5 files changed, 4 insertions(+), 54 deletions(-) delete mode 100755 hugegraph-loader/assembly/travis/install-hugegraph-from-tar.sh diff --git a/hugegraph-client/README.md b/hugegraph-client/README.md index d2360c1a2..004f112c2 100644 --- a/hugegraph-client/README.md +++ b/hugegraph-client/README.md @@ -5,7 +5,7 @@ [![codecov](https://codecov.io/gh/hugegraph/hugegraph-client/branch/master/graph/badge.svg)](https://codecov.io/gh/hugegraph/hugegraph-client) [![Maven Central](https://img.shields.io/maven-central/v/org.apache.hugegraph/hugegraph-client)](https://mvnrepository.com/artifact/org.apache.hugegraph/hugegraph-client) -hugegraph-client is a Java-written client of [HugeGraph](https://github.com/hugegraph/hugegraph), providing operations of graph, schema, gremlin, variables and traversals etc. All these operations are interpreted and translated into RESTful requests to HugeGraph Server. Besides, hugegraph-client also checks arguments, serializes and deserializes structures and encapsulates server exceptions. +hugegraph-client is a Java-written client of [HugeGraph](https://github.com/apache/hugegraph), providing operations of graph, schema, gremlin, variables and traversals etc. All these operations are interpreted and translated into RESTful requests to HugeGraph Server. Besides, hugegraph-client also checks arguments, serializes and deserializes structures and encapsulates server exceptions. ## Features diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/util/IdUtil.java b/hugegraph-client/src/main/java/org/apache/hugegraph/util/IdUtil.java index 7934dfe6c..899c39ced 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/util/IdUtil.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/util/IdUtil.java @@ -20,7 +20,7 @@ import org.apache.commons.lang3.StringUtils; /** - * Copied from HugeGraph(...) + * Copied from HugeGraph(...) */ public final class IdUtil { diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/util/SplicingIdGenerator.java b/hugegraph-client/src/main/java/org/apache/hugegraph/util/SplicingIdGenerator.java index 6acb6493e..9bbf36882 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/util/SplicingIdGenerator.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/util/SplicingIdGenerator.java @@ -21,7 +21,7 @@ import java.util.List; /** - * Copied from HugeGraph(...) + * Copied from HugeGraph(...) */ public class SplicingIdGenerator { diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/test_configure_gremlin_fixture.py b/hugegraph-hubble/hubble-dist/assembly/travis/test_configure_gremlin_fixture.py index a9c635a4b..2e351af42 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/test_configure_gremlin_fixture.py +++ b/hugegraph-hubble/hubble-dist/assembly/travis/test_configure_gremlin_fixture.py @@ -76,10 +76,9 @@ def test_archive_commit_and_hash_bind_the_supplied_payload(self): output = root / "output" output.mkdir() commit = "1" * 40 - # TODO(apache/hugegraph#3263): align this temporary source fixture with ASF after the merge. env = dict(os.environ, SERVER_ARCHIVE=str(archive), SERVER_ARCHIVE_COMMIT=commit, SERVER_ARCHIVE_SHA256=hashlib.sha256(archive.read_bytes()).hexdigest(), - SERVER_REPOSITORY="hugegraph/hugegraph") + SERVER_REPOSITORY="apache/hugegraph") result = subprocess.run(["bash", str((HERE / "download-hugegraph.sh").resolve()), commit], cwd=output, env=env, capture_output=True) self.assertEqual(0, result.returncode, result.stderr.decode()) diff --git a/hugegraph-loader/assembly/travis/install-hugegraph-from-tar.sh b/hugegraph-loader/assembly/travis/install-hugegraph-from-tar.sh deleted file mode 100755 index 07a4bb335..000000000 --- a/hugegraph-loader/assembly/travis/install-hugegraph-from-tar.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with this -# work for additional information regarding copyright ownership. The ASF -# licenses this file to You under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -set -ev - -if [[ $# -ne 1 ]]; then - echo "Must input an existing commit id of hugegraph server" && exit 1 -fi - -VERSION=$1 -HUGEGRAPH_LINK="https://github.com/hugegraph/hugegraph/releases/download/v${VERSION}/hugegraph-${VERSION}.tar.gz" - -wget "${HUGEGRAPH_LINK}" || exit 1 -tar zxvf hugegraph-${VERSION}.tar.gz - -HTTPS_SERVER_DIR="hugegraph_https" -mkdir ${HTTPS_SERVER_DIR} -cp -r hugegraph-${VERSION}/. ${HTTPS_SERVER_DIR} -cd hugegraph-${VERSION} - -# start HugeGraphServer with http protocol -bin/init-store.sh || exit 1 -bin/start-hugegraph.sh || exit 1 - -cd ../${HTTPS_SERVER_DIR} -REST_SERVER_CONFIG="conf/rest-server.properties" -GREMLIN_SERVER_CONFIG="conf/gremlin-server.yaml" -sed -i "s?http://127.0.0.1:8080?https://127.0.0.1:8443?g" "$REST_SERVER_CONFIG" -sed -i "s/#port: 8182/port: 8282/g" "$GREMLIN_SERVER_CONFIG" -echo "gremlinserver.url=http://127.0.0.1:8282" >> $REST_SERVER_CONFIG - -# start HugeGraphServer with https protocol -bin/init-store.sh -bin/start-hugegraph.sh -cd ../ From 0c198100a78225604f59da5d526f4cfe77e91331 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 22:41:49 +0800 Subject: [PATCH 13/38] fix(ci): bind SDK archive to its source directory - share one checkout root between install and archive lookup - preserve the upstream Server distribution layout - separate local directory names from repository identity --- .github/actions/setup-candidate-sdk/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index 3908e9ead..a9214b317 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -55,9 +55,10 @@ runs: run: | candidate_repo=$(mktemp -d "$RUNNER_TEMP/hugegraph-candidate-m2.XXXXXX") server_build=$(mktemp -d "$RUNNER_TEMP/hugegraph-candidate-build.XXXXXX") + source_dir="$server_build/source" bash "$GITHUB_WORKSPACE/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ - "$SERVER_COMMIT" "$server_build/hugegraph" "$candidate_repo" - archives=("$server_build"/apache/hugegraph-server/apache-hugegraph-*.tar.gz) + "$SERVER_COMMIT" "$source_dir" "$candidate_repo" + archives=("$source_dir"/hugegraph-server/apache-hugegraph-*.tar.gz) if [[ ${#archives[@]} -ne 1 || ! -f "${archives[0]}" ]]; then printf 'Expected one candidate server archive, found: %s\n' "${archives[*]}" >&2 exit 1 From 6e43606e642b9e8de5b7ff5285aabe82c0a9cf6d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 23:14:53 +0800 Subject: [PATCH 14/38] fix(ci): reuse verified Java 17 server build - build current fixtures through the SDK installer - retain legacy Java 11 package and runtime behavior - bind fixture caches to installer inputs - update audited jraft dependency version --- .../actions/setup-hugegraph-server/action.yml | 4 ++-- .../actions/setup-hugegraph-server/fixture.sh | 24 +++++++++++++++---- .../setup-hugegraph-server/manifest.py | 1 + .../scripts/dependency/known-dependencies.txt | 2 +- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup-hugegraph-server/action.yml b/.github/actions/setup-hugegraph-server/action.yml index d80ec1d96..2077e6519 100644 --- a/.github/actions/setup-hugegraph-server/action.yml +++ b/.github/actions/setup-hugegraph-server/action.yml @@ -42,7 +42,7 @@ runs: uses: actions/cache@v4 with: path: ~/hugegraph-fixture-cache - key: server-fixture-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.repository }}-${{ inputs.commit-id }}-jdk${{ inputs.java-version }}-${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} + key: server-fixture-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.repository }}-${{ inputs.commit-id }}-jdk${{ inputs.java-version }}-${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} - name: Prepare fixture or start service shell: bash env: @@ -50,7 +50,7 @@ runs: FIXTURE_REPOSITORY: ${{ inputs.repository }} FIXTURE_COMMIT: ${{ inputs.commit-id }} FIXTURE_REF: ${{ inputs.fetch-ref }} - FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} + FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} FIXTURE_PLATFORM: ${{ runner.os }}-${{ runner.arch }} FIXTURE_JAVA: ${{ inputs.java-version }} FIXTURE_JAVA_HOME: ${{ inputs.java-home }} diff --git a/.github/actions/setup-hugegraph-server/fixture.sh b/.github/actions/setup-hugegraph-server/fixture.sh index 9fc5f1850..60df1d70b 100644 --- a/.github/actions/setup-hugegraph-server/fixture.sh +++ b/.github/actions/setup-hugegraph-server/fixture.sh @@ -26,7 +26,12 @@ FIXTURE_DIR=${FIXTURE_DIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-fixture} FIXTURE_HELPER=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/manifest.py FIXTURE_BUILD_INPUTS=${FIXTURE_BUILD_INPUTS:-$(python3 "$FIXTURE_HELPER" inputs)} FIXTURE_PLATFORM=${FIXTURE_PLATFORM:-$(uname -sm)} -FIXTURE_CONFIG="package -DskipTests -Dmaven.javadoc.skip=true -ntp;schema=1;inputs=$FIXTURE_BUILD_INPUTS;platform=$FIXTURE_PLATFORM" +if [[ "$FIXTURE_JAVA" == 17 ]]; then + FIXTURE_BUILD_COMMAND=candidate-sdk-install +else + FIXTURE_BUILD_COMMAND='package -DskipTests -Dmaven.javadoc.skip=true -ntp' +fi +FIXTURE_CONFIG="$FIXTURE_BUILD_COMMAND;schema=1;inputs=$FIXTURE_BUILD_INPUTS;platform=$FIXTURE_PLATFORM" export FIXTURE_REPOSITORY FIXTURE_COMMIT FIXTURE_JAVA FIXTURE_CONFIG FIXTURE_DIR [[ "$FIXTURE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] [[ "$FIXTURE_COMMIT" =~ ^[0-9a-f]{40}$ ]] @@ -39,7 +44,7 @@ fixture_validate_java() { fixture_verify() { python3 "$FIXTURE_HELPER" verify; } fixture_build() { - local cache_key cache_dir source_dir archive + local cache_key cache_dir source_dir archive candidate_repo='' cache_key=$(printf '%s\n' "$FIXTURE_REPOSITORY" "$FIXTURE_COMMIT" "$FIXTURE_JAVA" "$FIXTURE_CONFIG" | shasum -a 256 | cut -d ' ' -f 1) cache_dir="${HOME}/hugegraph-fixture-cache/$cache_key" mkdir -p "$FIXTURE_DIR" @@ -51,9 +56,17 @@ fixture_build() { rm -f "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" fixture_validate_java source_dir="$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-source.XXXXXX")/source" - SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \ - bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/checkout-server.sh" "$FIXTURE_COMMIT" "$source_dir" - (cd "$source_dir" && mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp) + if [[ "$FIXTURE_JAVA" == 17 ]]; then + # Reuse the verified SDK build's same-source distribution and isolated repository. + candidate_repo=$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-sdk-m2.XXXXXX") + SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \ + bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ + "$FIXTURE_COMMIT" "$source_dir" "$candidate_repo" + else + SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \ + bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/checkout-server.sh" "$FIXTURE_COMMIT" "$source_dir" + (cd "$source_dir" && mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp) + fi archive=$(find "$source_dir/hugegraph-server" -maxdepth 1 -name 'apache-hugegraph-*.tar.gz' -print) [[ -n "$archive" && -f "$archive" ]] cp "$archive" "$FIXTURE_DIR/server.tar.gz" @@ -63,6 +76,7 @@ fixture_build() { mkdir -p "$cache_dir" cp "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" "$cache_dir/" rm -rf "$source_dir" + if [[ -n "$candidate_repo" ]]; then rm -rf "$candidate_repo"; fi } fixture_copy_archive() { fixture_validate_java diff --git a/.github/actions/setup-hugegraph-server/manifest.py b/.github/actions/setup-hugegraph-server/manifest.py index c7446cecf..c7eab2468 100644 --- a/.github/actions/setup-hugegraph-server/manifest.py +++ b/.github/actions/setup-hugegraph-server/manifest.py @@ -31,6 +31,7 @@ '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', + 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh', diff --git a/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-dist/scripts/dependency/known-dependencies.txt index ea0cfbc83..1e8dce0c0 100644 --- a/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -337,7 +337,7 @@ jna-5.12.1.jar joda-time-2.10.8.jar joni-2.1.11.jar jpam-1.1.jar -jraft-core-1.3.11.jar +jraft-core-1.3.14.jar jsch-0.1.55.jar json-1.8.jar json-io-2.5.1.jar From 97dcf24cf786a1afeec39ce255db10a202ba25ed Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 23:15:32 +0800 Subject: [PATCH 15/38] fix(ci): align current fixture identity test - record the Java 17 SDK build configuration - preserve historical Java 11 fixture identity - retain fixture and caller JDK isolation assertions --- .github/actions/setup-hugegraph-server/runtime_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index 91ce54282..8ef582267 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -96,7 +96,10 @@ def test_composite_step_isolates_fixture_jdk_from_caller(self): for fixture_version, fixture_jdk, caller_version, caller_jdk in ( (11, jdk11, 17, jdk17), (17, jdk17, 11, jdk11)): with self.subTest(fixture=fixture_version, caller=caller_version): - self.manifest(java=str(fixture_version)) + config = self.env['FIXTURE_CONFIG'] + if fixture_version == 17: + config = 'candidate-sdk-install;' + config.split(';', 1)[1] + self.manifest(java=str(fixture_version), config=config) env = dict(self.env, JAVA_HOME=str(caller_jdk), PATH=f'{caller_jdk}/bin:{self.env["PATH"]}', FIXTURE_JAVA=str(fixture_version), FIXTURE_JAVA_HOME=str(fixture_jdk), From a8960909b1f4408cf3a311f89ba0c417121f442e Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 14:38:02 +0800 Subject: [PATCH 16/38] docs: clarify the SDK dependency build scope - describe the selected Server distribution and SDK modules - document Maven dependency closure installation - remove the outdated complete-reactor claim --- docs/java17-migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/java17-migration.md b/docs/java17-migration.md index 5b7f1df72..d1ac17e30 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -31,7 +31,7 @@ mvn -Dmaven.repo.local="$candidate_dir/m2" clean install \ -DskipTests -Dmaven.javadoc.skip=true -ntp ``` -The SDK helper installs the complete locked server reactor without starting a server. It requires a new external source directory and a dedicated Maven repository with no existing HugeGraph artifacts. It records source and artifact provenance in `candidate-sdk-manifest.json` inside that repository. Reuse the same explicit `-Dmaven.repo.local` option for subsequent Toolchain validation. The manifest's source and JDK paths describe that build; use your own directories when rebuilding. The package command skips test execution; run the module test suites separately. These source-built packages are candidates, not an ASF release. +The SDK helper installs the locked Server distribution, PD client, Store client and Struct modules together with their Maven dependency closure, without starting a server. Unrelated Server modules are excluded. It requires a new external source directory and a dedicated Maven repository with no existing HugeGraph artifacts. It records source and artifact provenance in `candidate-sdk-manifest.json` inside that repository. Reuse the same explicit `-Dmaven.repo.local` option for subsequent Toolchain validation. The manifest's source and JDK paths describe that build; use your own directories when rebuilding. The package command skips test execution; run the module test suites separately. These source-built packages are candidates, not an ASF release. Distribution packaging also requires Python 3.9 or newer. Before each module archive is written, the existing SDK verifier checks the locked source, required POM/JAR hashes and local installation origins, then checks the distribution's manifest and actual SDK libraries. The combined archive rechecks all three module directories before moving them. Source checks lock Server SDK artifacts; Toolchain artifacts such as the Client, Loader and Hubble backend are outputs of this reactor and may change during `install`. Older manifests can retain their metadata, but those outputs are excluded from SDK source hashes. The original manifest is still retained unchanged and compared with each packaged copy. A missing manifest, a Central artifact with the same version, or a stale/changed packaged SDK library fails packaging. Ordinary `compile` and `test` do not require the candidate manifest; they may validate published dependencies without establishing candidate package identity. From 59ef0cf3a5b27b832b034ae4b04d81b986879585 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 16:20:27 +0800 Subject: [PATCH 17/38] fix(ci): fetch immutable ASF SDK baseline - fetch the verified ASF commit instead of moving master - retain full source identity and artifact provenance checks - document the fixed baseline separately from latest master --- .github/actions/setup-candidate-sdk/action.yml | 4 ++-- .github/workflows/ci.yml | 2 +- docs/ci.md | 2 +- docs/java17-migration.md | 3 +-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index a9214b317..db62765e5 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -6,8 +6,8 @@ inputs: description: 'GitHub owner/repository for the candidate SDK' default: 'apache/hugegraph' ref: - description: 'Fetchable candidate source ref' - default: 'master' + description: 'Immutable candidate source commit' + default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' expected-commit: description: 'Locked full candidate source commit' default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3736c5019..c45a0f502 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: uses: ./.github/actions/get-hugegraph-commit with: repository: apache/hugegraph - ref: master + ref: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc expected-commit: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc - name: Record external test inputs env: diff --git a/docs/ci.md b/docs/ci.md index 2086dd157..7703d4ca1 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -47,7 +47,7 @@ module tests. They do not publish an image. Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement the existing affected-module selection without adding unrelated modules. -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. +The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. Required CI fetches the immutable baseline `d9abc` by its full commit, so later master updates do not change the build inputs. The released Server package and Hubble's candidate fixture are each built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. diff --git a/docs/java17-migration.md b/docs/java17-migration.md index d1ac17e30..8aaa46b12 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -6,7 +6,7 @@ Toolchain builds, tests and applications use Java 17. The released Server 1.7 co ## Build the locked candidate -The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses `apache/hugegraph` master with a verified ASF commit. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. +The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses the verified `apache/hugegraph` master baseline `d9abc`, fetching the immutable full commit instead of the moving branch. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. Use an explicit JDK directory and check both Java and Maven. On systems where Java 17 is not registered, a system JDK selector can return another installed version. Checking the generated class version alone does not identify the JVM that built or tested it. @@ -22,7 +22,6 @@ toolchain_root="$PWD" candidate_dir=$(mktemp -d /tmp/hugegraph-java17.XXXXXX) mkdir "$candidate_dir/m2" export SERVER_REPOSITORY=apache/hugegraph -export SERVER_FETCH_REF=master server_commit=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc bash "$toolchain_root/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ From b3011613fb03ed3332e3b72b0415d365ad70121b Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 16:26:36 +0800 Subject: [PATCH 18/38] fix(ci): resolve abbreviated source identities - resolve short Git selectors before transport and verification - retain complete machine identity and SDK artifact checks - bind packaging to the selected CI source context - shorten configuration and user-facing baseline references --- .../actions/get-hugegraph-commit/action.yml | 4 +- .../actions/get-hugegraph-commit/resolve.cjs | 26 +++++++++-- .../get-hugegraph-commit/resolve.test.cjs | 24 ++++++++++ .../actions/setup-candidate-sdk/action.yml | 6 +-- .github/scripts/test_candidate_sdk.py | 43 +++++++++++++++++- .github/scripts/verify_candidate_image_sdk.py | 35 +++++++++++++-- .github/workflows/ci.yml | 4 +- docs/ci.md | 2 +- docs/java17-migration.md | 4 +- .../assembly/travis/checkout-server.sh | 45 +++++++++++++++++-- .../assembly/travis/install-candidate-sdk.sh | 3 ++ .../assembly/travis/install-server.test.cjs | 19 ++++++++ 12 files changed, 193 insertions(+), 22 deletions(-) diff --git a/.github/actions/get-hugegraph-commit/action.yml b/.github/actions/get-hugegraph-commit/action.yml index b07b45c9e..df3d4c0a1 100644 --- a/.github/actions/get-hugegraph-commit/action.yml +++ b/.github/actions/get-hugegraph-commit/action.yml @@ -13,7 +13,7 @@ inputs: description: 'Branch, tag or commit; mutually exclusive with server-pr-number' default: '' expected-commit: - description: 'Optional full commit SHA; fail if the selected ref has moved' + description: 'Optional six-character or full commit SHA; fail if the selected ref has moved' default: '' outputs: @@ -47,7 +47,7 @@ runs: core.exportVariable('COMMIT_REF', result.fetchRef); core.setOutput('commit_id', result.sha); core.setOutput('fetch_ref', result.fetchRef); - core.info(`Server baseline: ${process.env.SERVER_REPOSITORY} ${result.fetchRef} (${result.sha})`); + core.info(`Server baseline: ${process.env.SERVER_REPOSITORY} ${/^[0-9a-f]{40}$/i.test(result.fetchRef) ? result.fetchRef.slice(0, 6) : result.fetchRef} (${result.sha.slice(0, 6)})`); } catch (error) { core.setFailed(`Failed to get HugeGraph commit: ${error.message}`); } diff --git a/.github/actions/get-hugegraph-commit/resolve.cjs b/.github/actions/get-hugegraph-commit/resolve.cjs index 050637950..556f986d5 100644 --- a/.github/actions/get-hugegraph-commit/resolve.cjs +++ b/.github/actions/get-hugegraph-commit/resolve.cjs @@ -19,8 +19,8 @@ module.exports = async function resolve(github, { repository, ref = '', pr = '', if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(repository)) { throw new Error('Invalid server repository'); } - if (expectedCommit && !/^[0-9a-f]{40}$/i.test(expectedCommit)) { - throw new Error('Expected commit must be a full SHA'); + if (expectedCommit && !/^(?:[0-9a-f]{6}|[0-9a-f]{40})$/i.test(expectedCommit)) { + throw new Error('Expected commit must be a six-character or full SHA'); } if (ref && pr) throw new Error('Select either ref or server-pr-number'); const [owner, repo] = repository.split('/'); @@ -45,8 +45,26 @@ module.exports = async function resolve(github, { repository, ref = '', pr = '', sha = commit.sha; } if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error('Server API returned an invalid commit SHA'); - if (expectedCommit && sha.toLowerCase() !== expectedCommit.toLowerCase()) { - throw new Error(`Server ref moved: expected ${expectedCommit}, resolved ${sha}`); + if (expectedCommit) { + let expectedSha = expectedCommit; + if (expectedCommit.length === 6) { + // Resolve independently: a matching prefix alone does not establish identity. + const { data: expected } = await github.rest.repos.getCommit({ owner, repo, ref: expectedCommit }); + expectedSha = expected.sha; + if (!/^[0-9a-f]{40}$/i.test(expectedSha) || + !expectedSha.toLowerCase().startsWith(expectedCommit.toLowerCase())) { + throw new Error('Server API returned an invalid expected commit SHA'); + } + } + if (sha.toLowerCase() !== expectedSha.toLowerCase()) { + throw new Error(`Server ref moved: expected ${expectedCommit.slice(0, 6)}, resolved ${sha.slice(0, 6)}`); + } + } + if (/^(?:[0-9a-f]{6}|[0-9a-f]{40})$/i.test(ref)) { + if (!sha.toLowerCase().startsWith(ref.toLowerCase())) { + throw new Error('Server API returned a commit outside the selected prefix'); + } + fetchRef = sha; } return { sha, fetchRef }; }; diff --git a/.github/actions/get-hugegraph-commit/resolve.test.cjs b/.github/actions/get-hugegraph-commit/resolve.test.cjs index 788be95a5..aa1383cd7 100644 --- a/.github/actions/get-hugegraph-commit/resolve.test.cjs +++ b/.github/actions/get-hugegraph-commit/resolve.test.cjs @@ -71,3 +71,27 @@ test('fails closed for a moved branch or API error', async () => { github.rest.repos.getCommit = async () => { throw new Error('Not Found'); }; await assert.rejects(resolve(github, { repository: 'example/server', ref: 'missing' }), /Not Found/); }); + +test('resolves an abbreviated commit to a complete fetch identity', async () => { + const { github, calls } = client(); + const short = sha.slice(0, 6); + assert.deepEqual(await resolve(github, { repository: 'apache/hugegraph', + ref: short, expectedCommit: short }), { sha, fetchRef: sha }); + assert.ok(calls.every(([, args]) => args.ref === short)); +}); + +test('checks complete identity even when resolved commits share the short prefix', async () => { + const { github } = client(); + github.rest.repos.getCommit = async ({ ref }) => ({ data: { + sha: ref === 'master' ? sha : sha.slice(0, 6) + 'b'.repeat(34) + } }); + await assert.rejects(resolve(github, { repository: 'apache/hugegraph', + ref: 'master', expectedCommit: sha.slice(0, 6) }), /Server ref moved/); +}); + +test('rejects abbreviated commits that GitHub cannot resolve', async () => { + const { github } = client(); + github.rest.repos.getCommit = async () => { throw new Error('Ambiguous commit'); }; + await assert.rejects(resolve(github, { repository: 'apache/hugegraph', + ref: sha.slice(0, 6), expectedCommit: sha.slice(0, 6) }), /Ambiguous commit/); +}); diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index db62765e5..552c3afe8 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -7,10 +7,10 @@ inputs: default: 'apache/hugegraph' ref: description: 'Immutable candidate source commit' - default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' + default: 'd9abcd' expected-commit: - description: 'Locked full candidate source commit' - default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' + description: 'Locked candidate source commit' + default: 'd9abcd' outputs: repository: diff --git a/.github/scripts/test_candidate_sdk.py b/.github/scripts/test_candidate_sdk.py index 5cbb88290..ff095aa93 100644 --- a/.github/scripts/test_candidate_sdk.py +++ b/.github/scripts/test_candidate_sdk.py @@ -19,6 +19,8 @@ import copy import json +import os +from unittest.mock import patch from pathlib import Path import shutil import subprocess @@ -29,6 +31,12 @@ class CandidateDistributionTest(unittest.TestCase): + def setUp(self): + self.source_commit = sdk.COMMIT + "a" * (40 - len(sdk.COMMIT)) + context = patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": self.source_commit}) + context.start() + self.addCleanup(context.stop) + def fixture(self, root): repository = root / "m2" artifacts, modules = [], [] @@ -49,7 +57,7 @@ def fixture(self, root): modules.append({"group_id": "org.apache.hugegraph", "artifact_id": artifact, "version": "1.7.0", "packaging": packaging, "source_pom": source_pom, "files": files}) - manifest = {"repository": sdk.REPOSITORY, "commit": sdk.COMMIT, + manifest = {"repository": sdk.REPOSITORY, "commit": self.source_commit, "source_revision": "1.7.0", "java_version": "17", "required_sdk_modules": modules, "artifacts": artifacts} self.write_manifest(repository, manifest) @@ -183,6 +191,39 @@ def test_reactor_installs_do_not_change_the_locked_sdk_contract(self): with self.assertRaisesRegex(RuntimeError, "SDK artifact hash mismatch"): sdk.validate_distribution(repository, directory, "hubble") + def test_short_source_context_preserves_full_identity_checks(self): + with tempfile.TemporaryDirectory() as temporary: + repository, manifest = self.fixture(Path(temporary)) + manifest["commit"] = sdk.COMMIT + "b" * (40 - len(sdk.COMMIT)) + self.write_manifest(repository, manifest) + with self.assertRaisesRegex(RuntimeError, "SDK source does not match"): + sdk.validate_sdk(repository) + with patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": sdk.COMMIT}): + with self.assertRaisesRegex(RuntimeError, "Invalid candidate SDK manifest"): + sdk.validate_sdk(repository) + + def test_explicit_resolved_context_supports_latest_master_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + repository, manifest = self.fixture(Path(temporary)) + selected = "b" * 40 + manifest["commit"] = selected + self.write_manifest(repository, manifest) + with patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": selected}): + sdk.validate_sdk(repository) + with self.assertRaisesRegex(RuntimeError, "SDK source does not match"): + sdk.validate_sdk(repository) + + def test_standalone_packaging_resolves_the_short_lock(self): + with tempfile.TemporaryDirectory() as temporary: + repository, _ = self.fixture(Path(temporary)) + with patch.dict(os.environ, {}, clear=True), patch.object(sdk, "_resolve_source_commit") as resolve: + resolve.return_value = self.source_commit + sdk.validate_sdk(repository) + resolve.assert_called_once_with("apache/hugegraph", sdk.COMMIT) + resolve.side_effect = OSError("ambiguous or inaccessible") + with self.assertRaisesRegex(RuntimeError, "Invalid candidate SDK manifest"): + sdk.validate_sdk(repository) + def test_candidate_manifest_requires_the_supported_build_jdk(self): with tempfile.TemporaryDirectory() as temporary: repository, manifest = self.fixture(Path(temporary)) diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index 783f881a0..2a6deab28 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -20,10 +20,14 @@ import argparse import hashlib import json +import os +import re +from functools import lru_cache +import urllib.request from pathlib import Path REPOSITORY = "apache/hugegraph" -COMMIT = "d9abcd4317fb36128e7e4d209139ec7f3a28cdfc" +COMMIT = "d9abcd" REQUIRED_MODULES = { "pom.xml", "hugegraph-commons/pom.xml", "hugegraph-server/pom.xml", "hugegraph-pd/pom.xml", "hugegraph-store/pom.xml", @@ -51,6 +55,31 @@ def digest(path): return checksum.hexdigest() + +@lru_cache(maxsize=1) +def _resolve_source_commit(repository, commit): + headers = {"Accept": "application/vnd.github+json", "User-Agent": "hugegraph-toolchain-ci"} + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(f"https://api.github.com/repos/{repository}/commits/{commit}", headers=headers) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response)["sha"] + + +def source_commit(): + # The SDK action exports the full identity already verified by the source resolver. + # Standalone packaging resolves the configured short commit through GitHub instead. + expected = os.environ.get("CANDIDATE_SOURCE_COMMIT") + if expected is None: + expected = _resolve_source_commit(REPOSITORY, COMMIT) + if not isinstance(expected, str) or not expected.startswith(COMMIT): + raise ValueError("Invalid resolved Server source identity") + if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{40}", expected): + raise ValueError("Invalid resolved Server source identity") + return expected + + def validate_sdk(repository): try: return _validate_sdk(Path(repository)) @@ -62,7 +91,7 @@ def validate_sdk(repository): def _validate_sdk(repository): repository = Path(repository) manifest = json.loads((repository / "candidate-sdk-manifest.json").read_text()) - if (manifest["repository"], manifest["commit"]) != (REPOSITORY, COMMIT): + if (manifest["repository"], manifest["commit"]) != (REPOSITORY, source_commit()): raise RuntimeError("SDK source does not match the locked candidate") java_version = manifest["java_version"] if not isinstance(java_version, str) or not java_version.isdecimal(): @@ -164,4 +193,4 @@ def validate_distribution(repository, directory, module): validate_distribution(args.repository, args.distribution, args.module) except RuntimeError as error: parser.exit(1, str(error) + "\n") - print(f"Verified {manifest['repository']}@{manifest['commit']}: {len(jars)} candidate JARs") + print(f"Verified {manifest['repository']}@{manifest['commit'][:6]}: {len(jars)} candidate JARs") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c45a0f502..5872593bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,8 +86,8 @@ jobs: uses: ./.github/actions/get-hugegraph-commit with: repository: apache/hugegraph - ref: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc - expected-commit: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc + ref: d9abcd + expected-commit: d9abcd - name: Record external test inputs env: NEEDS_SERVER: ${{ steps.scope.outputs.needsFixture }} diff --git a/docs/ci.md b/docs/ci.md index 7703d4ca1..5be98d095 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -47,7 +47,7 @@ module tests. They do not publish an image. Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement the existing affected-module selection without adding unrelated modules. -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. Required CI fetches the immutable baseline `d9abc` by its full commit, so later master updates do not change the build inputs. +The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. Required CI fetches the immutable baseline `d9abcd` by resolving its six-character source selector to a complete commit, so later master updates do not change the build inputs. The released Server package and Hubble's candidate fixture are each built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. diff --git a/docs/java17-migration.md b/docs/java17-migration.md index 8aaa46b12..3b89ddb1d 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -6,7 +6,7 @@ Toolchain builds, tests and applications use Java 17. The released Server 1.7 co ## Build the locked candidate -The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses the verified `apache/hugegraph` master baseline `d9abc`, fetching the immutable full commit instead of the moving branch. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. +The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses the verified `apache/hugegraph` master baseline `d9abcd`, resolving the six-character commit through GitHub and fetching the complete immutable identity instead of the moving branch. Short prefixes that are ambiguous or inaccessible fail before fetching. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. Use an explicit JDK directory and check both Java and Maven. On systems where Java 17 is not registered, a system JDK selector can return another installed version. Checking the generated class version alone does not identify the JVM that built or tested it. @@ -22,7 +22,7 @@ toolchain_root="$PWD" candidate_dir=$(mktemp -d /tmp/hugegraph-java17.XXXXXX) mkdir "$candidate_dir/m2" export SERVER_REPOSITORY=apache/hugegraph -server_commit=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc +server_commit=d9abcd bash "$toolchain_root/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ "$server_commit" "$candidate_dir/server" "$candidate_dir/m2" diff --git a/hugegraph-client/assembly/travis/checkout-server.sh b/hugegraph-client/assembly/travis/checkout-server.sh index 53b543f0f..a4314f38e 100644 --- a/hugegraph-client/assembly/travis/checkout-server.sh +++ b/hugegraph-client/assembly/travis/checkout-server.sh @@ -26,12 +26,49 @@ COMMIT_ID=$1 CHECKOUT_DIR=$2 SERVER_REPOSITORY=${SERVER_REPOSITORY:-apache/hugegraph} SERVER_FETCH_REF=${SERVER_FETCH_REF:-$COMMIT_ID} -if [[ ! "$COMMIT_ID" =~ ^[0-9a-fA-F]{40}$ ]] || +if [[ ! "$COMMIT_ID" =~ ^([0-9a-fA-F]{6}|[0-9a-fA-F]{40})$ ]] || [[ ! "$SERVER_REPOSITORY" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*/[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then - echo "Expected a full commit SHA and GitHub owner/repository" >&2 + echo "Expected a six-character or full commit SHA and GitHub owner/repository" >&2 exit 1 fi +if [[ ${#COMMIT_ID} -eq 6 ]]; then + SHORT_COMMIT=$COMMIT_ID + COMMIT_ID=$(python3 - "$SERVER_REPOSITORY" "$SHORT_COMMIT" <<'PY_RESOLVE' +import json +import os +import re +import sys +import urllib.error +import urllib.request + +repository, short = sys.argv[1:] +headers = {'Accept': 'application/vnd.github+json', 'User-Agent': 'hugegraph-toolchain-ci'} +token = os.environ.get('GITHUB_TOKEN') or os.environ.get('GH_TOKEN') +if token: + headers['Authorization'] = 'Bearer ' + token +request = urllib.request.Request(f'https://api.github.com/repos/{repository}/commits/{short}', headers=headers) +try: + with urllib.request.urlopen(request, timeout=30) as response: + sha = json.load(response)['sha'] + if not isinstance(sha, str) or not re.fullmatch(r'[0-9a-fA-F]{40}', sha) or not sha.lower().startswith(short.lower()): + raise ValueError('Invalid commit identity') +except (OSError, ValueError, KeyError, TypeError): + raise SystemExit('Unable to resolve the abbreviated Server commit') from None +print(sha.lower()) +PY_RESOLVE + ) +fi +# Git transport requires a complete object ID, not an abbreviated commit prefix. +if [[ "$SERVER_FETCH_REF" =~ ^[0-9a-fA-F]{6}$ ]]; then + FETCH_PREFIX=$(printf '%s' "$SERVER_FETCH_REF" | tr '[:upper:]' '[:lower:]') + if [[ "$(printf '%s' "$COMMIT_ID" | tr '[:upper:]' '[:lower:]')" != "$FETCH_PREFIX"* ]]; then + echo "Server checkout mismatch: selected commit differs from fetch prefix" >&2 + exit 1 + fi + SERVER_FETCH_REF=$COMMIT_ID +fi + # Fetch a reachable ref, then verify it against the resolved SHA before building. # A moving branch or PR must fail closed rather than silently changing the baseline. [[ ! -e "$CHECKOUT_DIR" ]] || { echo "Checkout directory already exists" >&2; exit 1; } @@ -41,7 +78,7 @@ git -C "$CHECKOUT_DIR" fetch --depth 1 -- origin "$SERVER_FETCH_REF" git -C "$CHECKOUT_DIR" checkout --detach FETCH_HEAD ACTUAL_COMMIT=$(git -C "$CHECKOUT_DIR" rev-parse HEAD) if [[ "$ACTUAL_COMMIT" != "$(printf '%s' "$COMMIT_ID" | tr '[:upper:]' '[:lower:]')" ]]; then - echo "Server checkout mismatch: expected $COMMIT_ID, got $ACTUAL_COMMIT" >&2 + echo "Server checkout mismatch: expected ${COMMIT_ID:0:6}, got ${ACTUAL_COMMIT:0:6}" >&2 exit 1 fi -printf 'Server source: %s@%s\n' "$SERVER_REPOSITORY" "$ACTUAL_COMMIT" +printf 'Server source: %s@%s\n' "$SERVER_REPOSITORY" "${ACTUAL_COMMIT:0:6}" diff --git a/hugegraph-client/assembly/travis/install-candidate-sdk.sh b/hugegraph-client/assembly/travis/install-candidate-sdk.sh index fcc81d6b2..e4eff9a07 100755 --- a/hugegraph-client/assembly/travis/install-candidate-sdk.sh +++ b/hugegraph-client/assembly/travis/install-candidate-sdk.sh @@ -54,6 +54,9 @@ SDK_MODULES+=,hugegraph-store/hg-store-client,hugegraph-struct -Dflatten.mode=resolveCiFriendliesOnly -DupdatePomFile=true \ -DskipTests -Dmaven.javadoc.skip=true -ntp) ACTUAL_COMMIT=$(git -C "$SOURCE_DIR" rev-parse HEAD) +if [[ "$SERVER_FETCH_REF" =~ ^([0-9a-fA-F]{6}|[0-9a-fA-F]{40})$ ]]; then + export SERVER_FETCH_REF=$ACTUAL_COMMIT +fi python3 - "$SOURCE_DIR" "$CANDIDATE_REPO" "${SERVER_REPOSITORY:-apache/hugegraph}" \ "$ACTUAL_COMMIT" "$JAVA_HOME" "$JAVA_VERSION" <<'PY_MANIFEST' import hashlib diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs index 7c3fad105..9b74560f2 100644 --- a/hugegraph-client/assembly/travis/install-server.test.cjs +++ b/hugegraph-client/assembly/travis/install-server.test.cjs @@ -507,3 +507,22 @@ for (const scenario of [ assert.equal(readdirSync(root).some(name => name.startsWith('hugegraph-servers.')), false); }); } + +test('checkout resolves short source selectors before Git transport', t => { + const root = temp(t); + const bin = join(root, 'bin'); + mkdirSync(bin); + const full = 'a'.repeat(40); + const checkout = join(__dirname, 'checkout-server.sh'); + const calls = join(root, 'git-calls'); + writeFileSync(join(bin, 'python3'), `#!/bin/bash\ncat >/dev/null\nprintf '%s\\n' '${full}'\n`, { mode: 0o755 }); + writeFileSync(join(bin, 'git'), `#!/bin/bash\nprintf '%s\\n' "$*" >> "$GIT_CALLS"\nif [[ "$*" == *'rev-parse HEAD' ]]; then printf '%s\\n' '${full}'; fi\n`, { mode: 0o755 }); + const result = spawnSync('bash', [checkout, full.slice(0, 6), join(root, 'checkout')], { + env: { ...process.env, SERVER_REPOSITORY: 'apache/hugegraph', SERVER_FETCH_REF: full.slice(0, 6), + PATH: `${bin}:${process.env.PATH}`, GIT_CALLS: calls }, encoding: 'utf8' + }); + assert.equal(result.status, 0, result.stderr); + const transport = readFileSync(calls, 'utf8'); + assert.ok(transport.includes(`fetch --depth 1 -- origin ${full}\n`)); + assert.match(result.stdout, /Server source: apache\/hugegraph@aaaaaa/); +}); From 99d12e0f12efec8c29b7dc26b6c198fbfb72a232 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 16:27:12 +0800 Subject: [PATCH 19/38] fix(ci): shorten the release source selector - retain the historical release tag baseline - resolve its abbreviated expected commit through GitHub - compare the complete resolved source identity --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5872593bf..d9974fb41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: with: repository: apache/hugegraph ref: refs/tags/1.7.0 - expected-commit: b12425c2032bf0d21a97b8221f42a18055c2982f + expected-commit: b12425 - name: Resolve locked Java17 candidate baseline id: hubble-baseline if: steps.scope.outputs.needsFixture == 'true' From b3df4ddc4c0b2717bfef32b38b1acae729fe8ca9 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 16:27:26 +0800 Subject: [PATCH 20/38] feat(ci): check upstream master compatibility - resolve ASF master once for each compatibility run - build SDK and Server from the same resolved commit - reuse core compilation and existing unit and HTTP suites - retain diagnostic reports without changing required gates --- .github/workflows/upstream-compatibility.yml | 85 ++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/upstream-compatibility.yml diff --git a/.github/workflows/upstream-compatibility.yml b/.github/workflows/upstream-compatibility.yml new file mode 100644 index 000000000..524b92971 --- /dev/null +++ b/.github/workflows/upstream-compatibility.yml @@ -0,0 +1,85 @@ +name: Upstream compatibility + +on: + schedule: + - cron: '17 3 * * *' + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/upstream-compatibility.yml' + - '.github/actions/get-hugegraph-commit/**' + - '.github/actions/setup-candidate-sdk/**' + - 'hugegraph-client/assembly/travis/install-candidate-sdk.sh' + - 'hugegraph-client/assembly/travis/checkout-server.sh' + - '.github/scripts/verify_candidate_image_sdk.py' + +permissions: + contents: read + +concurrency: + group: upstream-compatibility-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + upstream-master: + if: github.event_name != 'schedule' || github.repository == 'apache/hugegraph-toolchain' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Fetch code + uses: actions/checkout@v7 + + - name: Setup Java environment + uses: ./.github/actions/setup-java-env + with: + java-version: '17' + distribution: temurin + + - name: Resolve latest ASF master once + id: upstream + uses: ./.github/actions/get-hugegraph-commit + with: + repository: apache/hugegraph + ref: master + + - name: Install SDK from the resolved commit + id: sdk + uses: ./.github/actions/setup-candidate-sdk + with: + repository: apache/hugegraph + ref: ${{ steps.upstream.outputs.commit_id }} + expected-commit: ${{ steps.upstream.outputs.commit_id }} + + - name: Compile core consumers + run: | + mvn install -pl hugegraph-client,hugegraph-loader,hugegraph-tools,hugegraph-spark-connector -am -DskipTests -Dmaven.javadoc.skip=true -ntp + mvn compile -pl hugegraph-hubble/hubble-be -Dmaven.javadoc.skip=true -ntp + + - name: Start the same-source Server fixture + env: + CANDIDATE_ARCHIVE: ${{ steps.sdk.outputs.archive }} + run: bash hugegraph-client/assembly/travis/start-hugegraph-servers.sh "$CANDIDATE_ARCHIVE" + + - name: Client unit and HTTP compatibility tests + working-directory: hugegraph-client + run: | + mvn test -Dtest=UnitTestSuite -ntp + mvn test -Dtest=ApiTestSuite -ntp + mvn test -Dtest=FuncTestSuite -ntp + + - name: Loader and Hubble backend unit tests + run: | + mvn test -P unit -pl hugegraph-loader -ntp + mvn test -P unit-test -pl hugegraph-hubble/hubble-be -ntp + + - name: Tools functional compatibility tests + run: mvn test -Dtest=FuncTestSuite -Dtools.test.disposable=true -pl hugegraph-tools -ntp + + - name: Retain test reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: upstream-compatibility-reports-${{ github.run_attempt }} + path: '**/target/surefire-reports/**' + if-no-files-found: warn + retention-days: 7 From 07c78f516f3add652b2b28f876d647065a1784c9 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 16:29:27 +0800 Subject: [PATCH 21/38] docs(ci): describe upstream compatibility checks - separate the fixed baseline from current master testing - document one resolved source identity per compatibility run - reuse existing core tests without duplicate engine matrices --- docs/ci.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ci.md b/docs/ci.md index 5be98d095..40b4b904c 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -104,3 +104,5 @@ Validate policy and retry behavior locally: python3 -m unittest discover -s .github/scripts -p 'test_*.py' actionlint ``` + +Upstream compatibility runs daily and on demand against the latest ASF master, resolved once to an immutable commit for each run. It reuses the SDK build, core compilation, Client HTTP tests, Loader and Hubble backend unit tests, and Tools functional tests; Hadoop, Spark/Flink engine matrices and frontend browser tests remain in module CI. This independent check does not change the locked Required CI baseline. From e0cfe882b55df232c23a8492efa817f91f431668 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 16:33:50 +0800 Subject: [PATCH 22/38] fix(images): shorten the locked source selector - select the immutable ASF baseline with six characters - resolve the full identity before fetching Server - remove the moving master fetch override --- hugegraph-hubble/Dockerfile | 3 +-- hugegraph-loader/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hugegraph-hubble/Dockerfile b/hugegraph-hubble/Dockerfile index 6ed45b30f..a8e4d6988 100644 --- a/hugegraph-hubble/Dockerfile +++ b/hugegraph-hubble/Dockerfile @@ -39,9 +39,8 @@ COPY .github/scripts/verify_candidate_image_sdk.py /pkg/.github/scripts/ RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ SERVER_REPOSITORY=apache/hugegraph \ - SERVER_FETCH_REF=master \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ + d9abcd /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 diff --git a/hugegraph-loader/Dockerfile b/hugegraph-loader/Dockerfile index 2c5ec396a..bb541981b 100644 --- a/hugegraph-loader/Dockerfile +++ b/hugegraph-loader/Dockerfile @@ -35,9 +35,8 @@ COPY .github/scripts/verify_candidate_image_sdk.py /pkg/.github/scripts/ RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ SERVER_REPOSITORY=apache/hugegraph \ - SERVER_FETCH_REF=master \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ + d9abcd /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 From 8bdace53c5bc0745735b31f7261272b9d1e2a16c Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 18:02:23 +0800 Subject: [PATCH 23/38] fix(ci): make coverage upload advisory - warn instead of failing when Codecov upload errors - keep upload setup failures outside test job results - reuse the existing upload steps and authentication --- .github/actions/upload-coverage/action.yml | 2 +- .github/workflows/client-ci.yml | 1 + .github/workflows/hubble-ci.yml | 1 + .github/workflows/loader-ci.yml | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml index 6d742ea54..bedc40b08 100644 --- a/.github/actions/upload-coverage/action.yml +++ b/.github/actions/upload-coverage/action.yml @@ -18,6 +18,6 @@ runs: with: use_oidc: true disable_search: true - fail_ci_if_error: true + fail_ci_if_error: false slug: ${{ github.repository }} files: ${{ inputs.file }} diff --git a/.github/workflows/client-ci.yml b/.github/workflows/client-ci.yml index f0f1f32f2..017644c49 100644 --- a/.github/workflows/client-ci.yml +++ b/.github/workflows/client-ci.yml @@ -83,6 +83,7 @@ jobs: mvn test -Dtest=FuncTestSuite - name: Upload coverage to Codecov + continue-on-error: true uses: ./.github/actions/upload-coverage with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index 0ac67b871..78e541e28 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -249,6 +249,7 @@ jobs: retention-days: 14 - name: Upload coverage to Codecov + continue-on-error: true uses: ./.github/actions/upload-coverage with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index d0ec8173b..1acde1eec 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -100,6 +100,7 @@ jobs: mvn test -P kafka - name: Upload coverage to Codecov + continue-on-error: true uses: ./.github/actions/upload-coverage with: token: ${{ secrets.CODECOV_TOKEN }} From 7786b05bad4ec42435a86e1060e482b643d47378 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 18:07:48 +0800 Subject: [PATCH 24/38] chore: make coverage thresholds advisory Move Codecov configuration to the repository root. Keep project and patch coverage statuses informational. Preserve the client example exclusion relative to the root. --- .../assembly/travis/codecov.yml => codecov.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) rename hugegraph-client/assembly/travis/codecov.yml => codecov.yml (80%) diff --git a/hugegraph-client/assembly/travis/codecov.yml b/codecov.yml similarity index 80% rename from hugegraph-client/assembly/travis/codecov.yml rename to codecov.yml index 106e68f94..9d4c85f1f 100644 --- a/hugegraph-client/assembly/travis/codecov.yml +++ b/codecov.yml @@ -14,5 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true + ignore: - - "src/main/java/org/apache/hugegraph/example" + - "hugegraph-client/src/main/java/org/apache/hugegraph/example" From bb5763d28be9a13a4bcbc86757c52c14158e1da0 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 18:42:07 +0800 Subject: [PATCH 25/38] fix(ci): separate required and advisory checks (#791) Separate affected consumer tests from merge requirements so maintainers can decide whether failed or cancelled advisory checks need another run. --- .asf.yaml | 1 - .../actions/setup-candidate-sdk/action.yml | 7 +- .github/scripts/check-rerun.py | 2 + .github/scripts/ci-policy.py | 130 ++++++++++++++---- .github/scripts/test_candidate_sdk.py | 79 ++++++++++- .github/scripts/test_check_rerun.py | 9 +- .github/scripts/test_ci_policy.py | 57 +++++++- .github/scripts/verify_candidate_image_sdk.py | 5 +- .github/workflows/ci.yml | 49 +++---- .github/workflows/codeql-analysis.yml | 39 +++++- .github/workflows/dependency-license.yml | 57 ++++++++ .github/workflows/license-checker.yml | 55 +------- .github/workflows/loader-ci.yml | 20 +-- .github/workflows/rerun-ci.yml | 2 +- .github/workflows/server-fixture-ci.yml | 8 +- docs/ci.md | 120 +++++++++++----- docs/java17-migration.md | 13 +- .../assembly/travis/install-candidate-sdk.sh | 12 +- .../assembly/travis/install-server.test.cjs | 5 +- .../scripts/dependency/known-dependencies.txt | 2 +- hugegraph-hubble/Dockerfile | 7 +- .../travis/test_configure_gremlin_fixture.py | 3 +- hugegraph-loader/Dockerfile | 7 +- 23 files changed, 498 insertions(+), 191 deletions(-) create mode 100644 .github/workflows/dependency-license.yml diff --git a/.asf.yaml b/.asf.yaml index ee867caa4..e9e8aec36 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -45,7 +45,6 @@ github: # contexts are the names of checks that must pass contexts: - check-license-header - - affected-module-tests required_pull_request_reviews: dismiss_stale_reviews: true require_code_owner_reviews: false diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index 23e66b95f..b6652a8c1 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -1,17 +1,16 @@ name: 'Setup same-source candidate SDK' description: 'Build a locked Java 17 server reactor outside the workspace in an isolated Maven repository' -# TODO(apache/hugegraph#3263): temporary pre-merge source; after merge use apache/hugegraph and a verified ASF SHA. inputs: repository: description: 'GitHub owner/repository for the candidate SDK' - default: 'hugegraph/hugegraph' + default: 'apache/hugegraph' ref: description: 'Fetchable candidate source ref' - default: 'codex/cypher-minimal-compat' + default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' expected-commit: description: 'Locked full candidate source commit' - default: '4c162f539b906fa06dd83228e3691b77fa5b77d7' + default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' outputs: repository: diff --git a/.github/scripts/check-rerun.py b/.github/scripts/check-rerun.py index 408b5655b..a02b28b92 100644 --- a/.github/scripts/check-rerun.py +++ b/.github/scripts/check-rerun.py @@ -38,6 +38,8 @@ def decide(repository, run_id, expected_attempt, max_reruns, fetch=api): return "skip", "source run changed or is no longer a completed failure" if expected_attempt > max_reruns: return "skip", "retry limit reached" + if run.get("path") != ".github/workflows/license-checker.yml": + return "skip", "automatic retry is limited to the required license workflow" sha = run.get("head_sha") if not sha: return "skip", "missing head SHA" diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index eb8302fbe..193efcc41 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -14,11 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Conservative affected-module selection and current-run CI gating.""" +"""Conservative module selection and truthful advisory CI reports.""" import argparse import json import os +import re from pathlib import Path import subprocess from urllib.parse import quote @@ -43,7 +44,7 @@ "toolchain": {"client-ci.yml": ["client"], "client-go-ci.yml": ["go"], "loader-ci.yml": ["loader"], "tools-ci.yml": ["tools"], "spark-connector-ci.yml": ["spark"], "hubble-ci.yml": ["hubble"], - "codeql-analysis.yml": []}, + "codeql-analysis.yml": [], "dependency-license.yml": [], "license-checker.yml": []}, } IMAGES = {"loader_image", "hubble_image"} @@ -149,6 +150,47 @@ def select_images(paths): return selected +def scan_languages(paths): + """Scan supported sources, keeping unknown/shared build inputs conservative.""" + all_languages = {"java", "javascript", "python"} + if not paths: + return all_languages + languages = set() + for path in paths: + p = Path(path) + if documentation(path) or image_only_path(path): + continue + if p.suffix == ".py": + languages.add("python") + elif p.suffix in {".js", ".jsx", ".ts", ".tsx", ".cjs", ".mjs"}: + languages.add("javascript") + elif p.suffix == ".java": + languages.add("java") + elif path.startswith("hugegraph-client-go/"): + continue # Go is not a configured CodeQL language in this repository. + elif path.startswith("hugegraph-hubble/hubble-fe/"): + languages.add("javascript") + elif path.startswith("hugegraph-hubble/hubble-be/"): + languages.add("java") + elif path.startswith("hugegraph-hubble/"): + languages.update({"java", "javascript"}) + elif any(path.startswith(prefix) for prefix in ( + "hugegraph-client/", "hugegraph-loader/", "hugegraph-tools/", + "hugegraph-spark-connector/")): + languages.add("java") + elif path.startswith(".github/workflows/") and p.name in WORKFLOWS["toolchain"]: + modules = WORKFLOWS["toolchain"][p.name] + if p.name == "codeql-analysis.yml": + languages.update(all_languages) + elif "hubble" in modules: + languages.update({"java", "javascript"}) + elif set(modules) - {"go"}: + languages.add("java") + else: + languages.update(all_languages) + return languages + + def select(project, paths): selected = set() for path in paths: @@ -245,8 +287,6 @@ def require_current_pr(plan, fetch): or ref.get("object", {}).get("sha") != plan["base"] or live.get("merge_commit_sha") != plan["testedMergeSHA"]): raise StaleInputError("PR stack or canonical base changed; start a new PR run") - elif live["base"]["sha"] != plan["base"]: - raise StaleInputError("PR base changed; start a new PR run") def record_stack_merge(plan, event_pr, parents, fetch): @@ -286,6 +326,7 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): "reason": "affected inputs from the cumulative PR diff", "externalInputs": external_inputs or {}, "testedMergeSHA": git("rev-parse", "HEAD")} selected_images = set() + conservative_scan = False try: pr = event.get("pull_request") if "pull_request" in event: @@ -315,12 +356,15 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): selected_images = select_images(paths) if project == "toolchain" else set() if unsafe_documentation(plan["base"], paths) or unsafe_documentation(plan["testedMergeSHA"], paths): selected = set(MODULES[project]) + conservative_scan = True if project == "toolchain" and packaged_readme_state(plan["testedMergeSHA"]) != "valid": selected = set(MODULES[project]) + conservative_scan = True except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): selected = set(MODULES[project]) selected_images = set(IMAGES) if project == "toolchain" else set() - plan["reason"] = "selection unavailable: full required coverage" + plan["reason"] = "selection unavailable: full validation coverage" + conservative_scan = True if not selected and not selected_images: plan["reason"] = "cumulative PR diff contains only plain prose documentation; modules unaffected" plan["selected"] = sorted(selected) @@ -334,12 +378,62 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): plan["needsFixture"] = project == "toolchain" and any(plan.get(m, False) for m in MODULES[project]) plan["fixture"] = plan["needsFixture"] plan["compatibility"] = project == "server" and bool(selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"})) - plan["security"] = bool(selected) or any(p.startswith(".github/workflows/codeql") for p in locals().get("paths", [])) - plan["security_languages"] = json.dumps(["java"]) + paths = locals().get("paths", []) + plan["changedPaths"] = paths + plan["selectionReasons"] = { + suite: [path for path in paths if suite in (suites(project, select(project, [path])) + + list(select_images([path]) if project == "toolchain" else []))] + for suite in plan["expected"] + } + plan["dependency_audit"] = (not paths or any(dependency_input(path) for path in paths)) + if project == "toolchain" and plan["dependency_audit"]: + plan["expected"].append("dependency-audit") + plan["selectionReasons"]["dependency-audit"] = [path for path in paths if dependency_input(path)] + languages = scan_languages([] if conservative_scan else paths) + plan["security"] = bool(languages) + plan["security_languages"] = json.dumps(sorted(languages)) + write_summary(plan) return plan +def dependency_input(path): + return bool(re.search(r"(^|/)(pom\.xml|package\.json|yarn\.lock|package-lock\.json)$|" + r"(^|/)(assembly|licenses)/|(^|/)(LICENSE|NOTICE)$|dependency/|" + r"^\.mvn/|^\.github/(configs/|workflows/(license-checker|dependency-license)\.yml)", path)) + + +def write_summary(plan, results=None): + summary = "## Module validation (advisory)\n\n" + summary += "Required: **check-license-header** only. Module, image and scan results do not block merging.\n\n" + summary += plan.get("reason", "affected inputs") + "\n\n" + summary += "| Check | Required | Selection reason | Result |\n| --- | --- | --- | --- |\n" + for suite in sorted(set(MODULES[plan["project"]]) | IMAGES | {"dependency-audit"}): + selected = suite in plan["expected"] + reason = ", ".join(plan.get("selectionReasons", {}).get(suite, [])) or ( + plan.get("reason", "affected inputs") if selected else "unaffected") + reason = reason.replace("|", "\\|").replace("\n", " ") + state = results.get(suite, {}).get("result", "missing") if results is not None and selected else ( + "scheduled" if selected else "not selected") + summary += f"| {suite} | No | {reason} | {state} |\n" + summary += f"\nTested head: `{plan['head']}`; base: `{plan['base']}`; checkout: `{plan['testedMergeSHA']}`.\n" + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream: + stream.write(summary) + + +def result_report(plan, results): + report = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", + "head", "testedMergeSHA", "externalInputs"]} + if "stack" in plan: + report.update(baseBranch=plan["baseBranch"], stack=plan["stack"]) + report.update(runID=int(os.environ.get("GITHUB_RUN_ID", "0")), + runAttempt=int(os.environ.get("GITHUB_RUN_ATTEMPT", "1")), + executed=plan["expected"], results=results) + return report + + def gate(plan, results, fetch=None): + write_summary(plan, results) if results.get("plan", {}).get("result") != "success": raise ValueError("planner did not succeed") for suite in plan["expected"]: @@ -352,21 +446,7 @@ def gate(plan, results, fetch=None): if "hubble" in plan["expected"] and results.get("hubble-fixture", {}).get("result") != "success": raise ValueError("selected Hubble tests lack successful baseline fixture") require_current_pr(plan, fetch or api) - report = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", - "head", "testedMergeSHA", "externalInputs"]} - if "stack" in plan: - report.update(baseBranch=plan["baseBranch"], stack=plan["stack"]) - report.update(runID=int(os.environ.get("GITHUB_RUN_ID", "0")), - runAttempt=int(os.environ.get("GITHUB_RUN_ATTEMPT", "1")), - executed=plan["expected"], results=results) - # These fields describe this run; they do not authorize reuse by another run. - summary = "Selection: " + plan.get("reason", "affected inputs") + "\nExecuted: " + ", ".join(plan["expected"]) - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: - out.write("## Affected module tests\n" + summary + "\n\nUnselected modules: " - + ", ".join(sorted(set(MODULES[plan["project"]]) - set(plan["selected"]))) + "\n") - print(summary) - return report + return result_report(plan, results) def main(): @@ -394,7 +474,11 @@ def main(): output.write(f"{key}={str(item).lower()}\n") output.write(f"security_languages={value['security_languages']}\nplan_file={args.output}\n") else: - value = gate(json.loads(Path(args.plan).read_text()), json.loads(Path(args.results).read_text())) + plan = json.loads(Path(args.plan).read_text()) + results = json.loads(Path(args.results).read_text()) + # Preserve diagnostic results even when an advisory job fails or is missing. + Path(args.output).write_text(json.dumps(result_report(plan, results), indent=2) + "\n") + value = gate(plan, results) Path(args.output).write_text(json.dumps(value, indent=2) + "\n") diff --git a/.github/scripts/test_candidate_sdk.py b/.github/scripts/test_candidate_sdk.py index ef152ffa4..7d27fc899 100644 --- a/.github/scripts/test_candidate_sdk.py +++ b/.github/scripts/test_candidate_sdk.py @@ -15,19 +15,96 @@ # License for the specific language governing permissions and limitations # under the License. # -"""Bounded SDK provenance fixtures; these do not build real Maven artifacts.""" +"""SDK provenance fixtures and an opt-in real Maven parent-metadata regression.""" import copy import json +import os from pathlib import Path import shutil import subprocess import tempfile import unittest +import uuid +import xml.etree.ElementTree as ET import verify_candidate_image_sdk as sdk +class CandidateParentModelTest(unittest.TestCase): + @unittest.skipUnless(os.environ.get("CANDIDATE_SDK_MAVEN_TEST"), + "Set CANDIDATE_SDK_MAVEN_TEST to a Maven executable for the metadata regression") + def test_license_metadata_can_rebuild_installed_root_parent(self): + # Use the real metadata plugin and unique coordinates, never the user's SDK artifacts. + maven = os.environ["CANDIDATE_SDK_MAVEN_TEST"] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + settings = root / "settings.xml" + settings.write_text("") + header = '4.0.0' + flatten = ('org.codehaus.mojoflatten-maven-plugin' + '1.2.7true' + 'resolveCiFriendliesOnly' + 'process-resourcesflatten' + '') + for fixed in (False, True): + group = "org.example.candidatesdk.g" + uuid.uuid4().hex + source = root / group + source.mkdir() + root_pom = (header + 'org.apacheapache' + '23' + group + '' + 'root${revision}pom' + '1.7.0dependency' + 'consumer' + 'maven-remote-resources-plugin3.3.0' + '') + (source / "pom.xml").write_text(root_pom) + parent = ('' + group + 'root' + '${revision}') + for module in ("dependency", "consumer"): + directory = source / module + directory.mkdir() + body = flatten if module == "dependency" else "" + dependency = (('' + group + '' + 'dependency1.7.0' + '') if module == "consumer" else "") + (directory / "pom.xml").write_text(header + parent + '' + module + '' + + dependency + '' + body + + '') + original_poms = {p: p.read_bytes() for p in source.rglob("pom.xml")} + + def run(*arguments): + return subprocess.run([maven, "--settings", str(settings), + "-Dmaven.repo.local=" + str(root / "m2"), "-B", "-ntp", *arguments], + cwd=source, capture_output=True, text=True, timeout=240) + + goals = (["org.codehaus.mojo:flatten-maven-plugin:1.2.7:flatten", "install", + "-Dflatten.mode=resolveCiFriendliesOnly", "-DupdatePomFile=true"] + if fixed else ["install"]) + literal_request = group + ":root:pom:${revision}" + installed = run(*goals, "-DskipTests") + if fixed or installed.returncode == 0: + self.assertEqual(0, installed.returncode, installed.stdout + installed.stderr) + else: + # Newer resolvers fail here; older ones only warn on the leaf metadata goal. + self.assertIn(literal_request, installed.stdout + installed.stderr) + metadata_goal = ("org.apache.maven.plugins:maven-remote-resources-plugin:3.3.0:" + "process@process-resource-bundles") + metadata = run(metadata_goal, "-f", "consumer/pom.xml", "-X") + output = metadata.stdout + metadata.stderr + if fixed: + self.assertEqual(0, metadata.returncode, output) + self.assertNotIn(literal_request, output) + self.assertNotIn("Failed to build parent project", output) + version = ET.parse(source / ".flattened-pom.xml").getroot().findtext( + "{http://maven.apache.org/POM/4.0.0}version") + self.assertEqual("1.7.0", version) + else: + self.assertIn(literal_request, output) + for path, original in original_poms.items(): + self.assertEqual(original, path.read_bytes()) + + class CandidateDistributionTest(unittest.TestCase): def fixture(self, root): repository = root / "m2" diff --git a/.github/scripts/test_check_rerun.py b/.github/scripts/test_check_rerun.py index 241210c74..70f5e7a68 100644 --- a/.github/scripts/test_check_rerun.py +++ b/.github/scripts/test_check_rerun.py @@ -32,7 +32,8 @@ class FreshnessTest(unittest.TestCase): def setUp(self): self.repository = "hugegraph/hugegraph-toolchain" self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1, - "event": "push", "head_sha": "old", "head_branch": "release/test"} + "event": "push", "head_sha": "old", "head_branch": "release/test", + "path": ".github/workflows/license-checker.yml"} self.head = "old" self.calls = [] @@ -81,6 +82,12 @@ def test_current_run_must_remain_failed_and_completed(self): self.assertEqual("skip", self.decide()) self.run[key] = old + def test_advisory_and_unknown_workflows_are_not_retried(self): + for path in [None, ".github/workflows/ci.yml", ".github/workflows/codeql-analysis.yml"]: + self.run["path"] = path + with self.subTest(path=path): + self.assertEqual("skip", self.decide()) + def test_retry_limit(self): self.run["run_attempt"] = 2 self.assertEqual("rerun", checker.decide(self.repository, 42, 2, 2, self.fetch)[0]) diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 196fd8bfd..77e22fd97 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -40,6 +40,51 @@ def setUp(self): self.addCleanup(api_mock.stop) self.api_mock = api_mock.start() + def test_codeql_uses_affected_supported_languages(self): + cases = [ + (["README.md", "docs/ci.md"], set()), + (["hugegraph-client-go/client.go"], set()), + ([".github/workflows/client-go-ci.yml"], set()), + (["hugegraph-loader/Dockerfile"], set()), + (["hugegraph-hubble/README.md"], set()), + ([".github/workflows/image-ci.yml"], set()), + (["hugegraph-client/src/A.java"], {"java"}), + (["hugegraph-loader/assembly/static/bin/hugegraph-loader.sh"], {"java"}), + (["hugegraph-hubble/hubble-fe/src/App.tsx"], {"javascript"}), + (["hugegraph-hubble/hubble-dist/assembly/static/conf/application.properties"], {"java", "javascript"}), + ([".github/scripts/helper.py"], {"python"}), + (["pom.xml"], {"java", "javascript", "python"}), + ([".mvn/maven.config"], {"java", "javascript", "python"}), + (["unknown-input"], {"java", "javascript", "python"}), + ] + for paths, expected in cases: + with self.subTest(paths=paths): + plan = self.pr_plan(paths) + self.assertEqual(expected, set(json.loads(plan["security_languages"]))) + self.assertEqual(bool(expected), plan["security"]) + self.assertEqual({"java", "javascript", "python"}, policy.scan_languages([])) + + def test_shared_maven_config_selects_dependency_audit_and_consumers(self): + plan = self.pr_plan([".mvn/maven.config"]) + self.assertTrue(plan["dependency_audit"]) + self.assertEqual(set(policy.MODULES["toolchain"]), set(plan["selected"])) + self.assertEqual(policy.IMAGES, set(plan["selectedImages"])) + for path in ["hugegraph-client/pom.xml", ".github/configs/settings.xml"]: + self.assertTrue(policy.dependency_input(path)) + self.assertFalse(self.pr_plan(["docs/ci.md"])["dependency_audit"]) + + def test_failure_summary_is_written_before_advisory_failure(self): + with tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / "summary.md" + with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(summary)}): + with self.assertRaises(ValueError): + policy.gate(self.plan(), {"plan": {"result": "success"}, + "client": {"result": "cancelled"}}) + text = summary.read_text() + self.assertIn("check-license-header", text) + self.assertIn("| client | No |", text) + self.assertIn("cancelled", text) + def test_dependency_expansion(self): self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, policy.select("toolchain", ["hugegraph-client/src/A.java"])) @@ -354,7 +399,7 @@ def git(*args): results.update(plan={"result": "success"}, fixture={"result": "success"}, **{"hubble-fixture": {"result": "success"}}) self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: live)["executed"]) - for section, field, value in [("head", "sha", advanced_head), ("base", "sha", advanced_base), + for section, field, value in [("head", "sha", advanced_head), ("head", "ref", "other-branch"), ("head", "repo", {"full_name": "other/fork"}), ("base", "repo", {"full_name": "other/target"})]: @@ -364,6 +409,10 @@ def git(*args): policy.create_plan("toolchain", event, "apache/t", lambda _: changed) with self.assertRaises(policy.StaleInputError): policy.gate(plan, results, lambda _: changed) + changed_base = dict(live, base=dict(live["base"], sha=advanced_base)) + self.assertEqual(plan["expected"], policy.create_plan( + "toolchain", event, "apache/t", lambda _: changed_base)["expected"]) + self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: changed_base)["executed"]) with self.assertRaises(policy.StaleInputError): policy.gate(plan, results, lambda _: dict(live, state="closed")) git("checkout", "--detach", "-q", head) @@ -441,6 +490,12 @@ def fetch(path): self.assertEqual(canonical, report["base"]) self.assertEqual("candidate", report["baseBranch"]) self.assertEqual(live["stack"], report["stack"]) + # Advisory failures retain the same tested stack identity. + failed = policy.result_report(plan, {"loader": {"result": "cancelled"}}) + self.assertEqual(canonical, failed["base"]) + self.assertEqual("candidate", failed["baseBranch"]) + self.assertEqual(live["stack"], failed["stack"]) + self.assertEqual("cancelled", failed["results"]["loader"]["result"]) original = json.loads(json.dumps(live)) for name in ["missing_stack", "changed_head", "changed_branch", "changed_stack", "changed_merge", "changed_base_ref", "unrelated_anchor"]: diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index 0667cf743..f2ec28090 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -22,9 +22,8 @@ import json from pathlib import Path -# TODO(apache/hugegraph#3263): temporary pre-merge source; after merge use apache/hugegraph and a verified ASF SHA. -REPOSITORY = "hugegraph/hugegraph" -COMMIT = "4c162f539b906fa06dd83228e3691b77fa5b77d7" +REPOSITORY = "apache/hugegraph" +COMMIT = "d9abcd4317fb36128e7e4d209139ec7f3a28cdfc" REQUIRED_MODULES = { "pom.xml", "hugegraph-commons/pom.xml", "hugegraph-server/pom.xml", "hugegraph-pd/pom.xml", "hugegraph-store/pom.xml", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57709a2b1..784e75ae1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ # under the License. # -name: Required CI +name: Module validation on: pull_request: @@ -53,7 +53,7 @@ jobs: needsFixture: ${{ steps.select.outputs.needsFixture }} loader-image: ${{ steps.select.outputs.loader_image }} hubble-image: ${{ steps.select.outputs.hubble_image }} - security: ${{ steps.select.outputs.security }} + dependency-audit: ${{ steps.select.outputs.dependency_audit }} steps: - uses: actions/checkout@v7 with: @@ -85,10 +85,9 @@ jobs: if: steps.scope.outputs.needsFixture == 'true' uses: ./.github/actions/get-hugegraph-commit with: - # TODO(apache/hugegraph#3263): switch the temporary source to ASF after merge. - repository: hugegraph/hugegraph - ref: codex/cypher-minimal-compat - expected-commit: 4c162f539b906fa06dd83228e3691b77fa5b77d7 + repository: apache/hugegraph + ref: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc + expected-commit: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc - name: Record external test inputs env: NEEDS_SERVER: ${{ steps.scope.outputs.needsFixture }} @@ -150,6 +149,11 @@ jobs: with: java-version: '11' distribution: temurin + - name: Test isolated candidate parent metadata + if: needs.plan.outputs.client == 'true' || needs.plan.outputs.loader == 'true' || needs.plan.outputs.tools == 'true' || needs.plan.outputs.spark == 'true' || needs.plan.outputs.hubble == 'true' + env: + CANDIDATE_SDK_MAVEN_TEST: mvn + run: python3 -m unittest discover -s .github/scripts -p 'test_candidate_sdk.py' - name: Build reusable Server package uses: ./.github/actions/setup-hugegraph-server with: @@ -273,7 +277,7 @@ jobs: affected-module-tests: name: affected-module-tests - needs: [plan, fixture, hubble-fixture, client, loader, tools, spark, hubble, go, loader_image, hubble_image] + needs: [plan, fixture, hubble-fixture, client, loader, tools, spark, hubble, go, loader_image, hubble_image, dependency-audit] if: ${{ !cancelled() }} runs-on: ubuntu-latest steps: @@ -283,7 +287,7 @@ jobs: - uses: actions/download-artifact@v7 with: name: ci-plan - - name: Require every selected module and image to succeed + - name: Report selected advisory tests and images env: GH_TOKEN: ${{ github.token }} RESULTS: ${{ toJSON(needs) }} @@ -291,6 +295,7 @@ jobs: printf '%s' "$RESULTS" > results.json python3 .github/scripts/ci-policy.py gate --plan plan.json --results results.json --output test-results.json - name: Publish current-run test results + if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: ci-test-results @@ -298,27 +303,7 @@ jobs: overwrite: true retention-days: 7 - security: - needs: [plan, affected-module-tests] - permissions: - contents: read - actions: read - security-events: write - if: >- - !cancelled() && needs.plan.result == 'success' && - needs.affected-module-tests.result == 'success' && - (needs.plan.outputs.security == 'true' || - (github.event_name == 'pull_request' && vars.CI_OPTIMIZED_REQUIRED != 'true')) - uses: ./.github/workflows/codeql-analysis.yml - secrets: inherit - - legacy-java-analysis: - name: Analyze (java) - needs: security - if: ${{ !cancelled() && github.event_name == 'pull_request' && vars.CI_OPTIMIZED_REQUIRED != 'true' }} - runs-on: ubuntu-latest - steps: - - name: Require actual Java security analysis during migration - env: - SECURITY_RESULT: ${{ needs.security.result }} - run: test "$SECURITY_RESULT" = success + dependency-audit: + needs: plan + if: needs.plan.outputs.dependency-audit == 'true' + uses: ./.github/workflows/dependency-license.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b6aa2a344..ece728f77 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,12 +2,44 @@ name: "CodeQL" on: workflow_dispatch: - workflow_call: + pull_request: + push: + branches: [master, 'release-*'] schedule: - cron: '42 20 * * 6' +permissions: + contents: read + pull-requests: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: + plan: + runs-on: ubuntu-latest + outputs: + security: ${{ steps.select.outputs.security }} + languages: ${{ steps.select.outputs.security_languages }} + dependency-audit: ${{ steps.select.outputs.dependency_audit }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + - name: Select scan inputs without preparing fixtures + id: select + env: + GH_TOKEN: ${{ github.token }} + run: >- + python3 .github/scripts/ci-policy.py plan --project toolchain + --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" + --output scan-plan.json + analyze: + needs: plan + if: needs.plan.outputs.security == 'true' env: # Preserve the master scan identity when invoked by a reusable-workflow caller. CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql-analysis.yml:analyze @@ -21,7 +53,7 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'java', 'javascript', 'python' ] + language: ${{ fromJSON(needs.plan.outputs.languages) }} # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] steps: @@ -83,7 +115,8 @@ jobs: category: .github/workflows/codeql-analysis.yml:analyze/language:${{ matrix.language }} dependency-review: - if: github.event_name == 'pull_request' + needs: plan + if: github.event_name == 'pull_request' && needs.plan.outputs.dependency-audit == 'true' runs-on: ubuntu-latest steps: - name: 'Checkout Repository' diff --git a/.github/workflows/dependency-license.yml b/.github/workflows/dependency-license.yml new file mode 100644 index 000000000..b576eac63 --- /dev/null +++ b/.github/workflows/dependency-license.yml @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Dependency inventory audit + +on: + workflow_call: + +permissions: + contents: read + +jobs: + check-dependency-license: + runs-on: ubuntu-latest + env: + SCRIPT_DEPENDENCY: hugegraph-dist/scripts/dependency + steps: + - name: Checkout source + uses: actions/checkout@v7 + - name: Set up JDK 17 + uses: ./.github/actions/setup-java-env + with: + java-version: '17' + distribution: 'temurin' + - name: Use Node.js 16 + uses: actions/setup-node@v6 + with: + node-version: '16' + + - name: Install locked candidate SDK + uses: ./.github/actions/setup-candidate-sdk + + - name: mvn install + run: | + mvn install -DskipTests=true -ntp + - name: generate current dependencies + run: | + bash "$SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh" current-dependencies.txt + - name: check third dependencies + run: | + bash "$SCRIPT_DEPENDENCY/check_dependencies.sh" diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml index 01f0847c6..6730f5691 100644 --- a/.github/workflows/license-checker.yml +++ b/.github/workflows/license-checker.yml @@ -34,32 +34,11 @@ concurrency: jobs: check-license-header: runs-on: ubuntu-latest - outputs: - dependencies: ${{ steps.dependencies.outputs.changed }} steps: - name: Fetch Code uses: actions/checkout@v7 with: - fetch-depth: 0 - - name: Select dependency license audit - id: dependencies - env: - EVENT_NAME: ${{ github.event_name }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - changed=true - if [[ "$EVENT_NAME" == pull_request ]]; then - if git diff --name-only "$BASE_SHA...$HEAD_SHA" > "$RUNNER_TEMP/license-paths"; then - changed=false - if grep -Eq '(^|/)(pom.xml|package.json|yarn.lock|package-lock.json)$|(^|/)(assembly|licenses)/|(^|/)(LICENSE|NOTICE)$|dependency/|^\.github/(configs/|workflows/license-checker.yml)' "$RUNNER_TEMP/license-paths"; then - changed=true - fi - fi - fi - echo "changed=$changed" >> "$GITHUB_OUTPUT" - - name: Test automatic retry freshness rules - run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' + persist-credentials: false # More info could refer to: https://github.com/apache/skywalking-eyes - name: Check License Header uses: apache/skywalking-eyes@29bd646002b63bb4c0ed1648d1654c35fb8ff027 # main 2026-07-12 @@ -80,35 +59,3 @@ jobs: mvn apache-rat:check -ntp find ./ -name rat.txt -print0 | xargs -0 -I file cat file > merged-rat.txt grep "Binaries" merged-rat.txt -C 3 && cat merged-rat.txt - - check-dependency-license: - needs: check-license-header - if: needs.check-license-header.outputs.dependencies == 'true' - runs-on: ubuntu-latest - env: - SCRIPT_DEPENDENCY: hugegraph-dist/scripts/dependency - steps: - - name: Checkout source - uses: actions/checkout@v7 - - name: Set up JDK 17 - uses: ./.github/actions/setup-java-env - with: - java-version: '17' - distribution: 'temurin' - - name: Use Node.js 16 - uses: actions/setup-node@v6 - with: - node-version: '16' - - - name: Install locked candidate SDK - uses: ./.github/actions/setup-candidate-sdk - - - name: mvn install - run: | - mvn install -DskipTests=true -ntp - - name: generate current dependencies - run: | - bash "$SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh" current-dependencies.txt - - name: check third dependencies - run: | - bash "$SCRIPT_DEPENDENCY/check_dependencies.sh" diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 24a8ba9d3..676dd81a4 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -64,18 +64,13 @@ jobs: id: candidate-sdk uses: ./.github/actions/setup-candidate-sdk - - name: Cache Hadoop - uses: actions/cache@v6 - with: - path: ~/hadoop-3.3.6.tar.gz - key: ${{ runner.os }}-hadoop-3.3.6 - - name: Compile run: | mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp - - name: Prepare Hadoop service - run: bash "$TRAVIS_DIR/install-hadoop.sh" + - name: Prepare HDFS from the pinned Hadoop image + timeout-minutes: 18 + run: bash "$TRAVIS_DIR/start-hdfs-ci.sh" - name: Setup released Server fixture if: matrix.SERVER_BASELINE == 'release-1.7' @@ -105,6 +100,15 @@ jobs: mvn test -P jdbc mvn test -P kafka + - name: Collect HDFS logs and remove containers + if: ${{ always() }} + run: | + for service in namenode datanode; do + name="${HDFS_CONTAINER_PREFIX:-hugegraph-ci-hdfs}-$service" + docker logs "$name" || true + docker rm -f "$name" || true + done + - name: Upload coverage to Codecov uses: ./.github/actions/upload-coverage with: diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml index 5c1fe8d93..3c1ece17a 100644 --- a/.github/workflows/rerun-ci.yml +++ b/.github/workflows/rerun-ci.yml @@ -3,7 +3,7 @@ name: "Rerun CI" on: workflow_run: workflows: - - "Required CI" + - "license checker" types: - completed diff --git a/.github/workflows/server-fixture-ci.yml b/.github/workflows/server-fixture-ci.yml index 6a21cfd24..316990afc 100644 --- a/.github/workflows/server-fixture-ci.yml +++ b/.github/workflows/server-fixture-ci.yml @@ -19,7 +19,7 @@ name: "server-fixture-ci" -# Automatic PR validation runs these contracts in Required CI. +# Automatic PR validation runs these contracts in Module validation. on: workflow_dispatch: @@ -44,5 +44,11 @@ jobs: node --test .github/actions/get-hugegraph-commit/resolve.test.cjs \ hugegraph-client/assembly/travis/checkout-server.test.cjs \ hugegraph-client/assembly/travis/install-server.test.cjs + - uses: ./.github/actions/setup-java-env + with: + java-version: '17' + distribution: temurin - name: Test candidate SDK and distribution provenance + env: + CANDIDATE_SDK_MAVEN_TEST: mvn run: python3 -m unittest discover -s .github/scripts -p 'test_candidate_sdk.py' diff --git a/docs/ci.md b/docs/ci.md index 0b7fb14a8..5b826a783 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,9 +1,22 @@ # Continuous integration -Every PR runs license checks and `Required CI`. The stable -`affected-module-tests` check requires all selected Linux tests and image startup -checks to pass; a failed, cancelled or unexpectedly skipped selected job cannot -satisfy it. +Only `check-license-header` (headers and RAT) is required for merging. Execution +selection remains conservative: changed modules and their consumers run, but their +results are advisory. Maintainers decide whether to rerun or request changes. + +The independent workflows are: + +| Workflow | Purpose | Required | Cancellation | +| --- | --- | --- | --- | +| license checker | Basic license checks | Yes | Cancelling it blocks merging | +| Module validation | Consumers, images, fixtures and dependency inventory audit | No | Can be cancelled without cancelling license checks or CodeQL | +| CodeQL | Affected-input scans and dependency review; weekly full scan | No | Independent of module tests | + +`affected-module-tests` remains a truthful advisory summary under its existing +name for migration. It fails when a selected test, image or fixture fails, is +cancelled or unexpectedly skipped; it does not make these checks required. +Plan and result summaries show specific checks, triggering paths, required roles +and actual results. Native job checks provide the detailed logs. | Change | Tests | | --- | --- | @@ -45,17 +58,39 @@ assets. These checks validate image build, packaging, startup and public entry points; Loader data ingestion and Hubble browser workflows remain covered by module tests. They do not publish an image. -Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement the existing affected-module selection without adding unrelated modules. - -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked candidate Server runs on Java 17. TODO ([apache/hugegraph#3263](https://github.com/apache/hugegraph/pull/3263)): replace the temporary candidate source with verified ASF inputs after the Server migration merges, and revalidate the SDK/runtime matrix. - -The released Server package and Hubble's candidate fixture are each built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. - -The shared Java initialization action requires Maven 3.9 or newer and supplies explicit settings through `MAVEN_ARGS`, preserving existing arguments and user settings. Dependencies are resolved from Central before ASF Stage, with Stage snapshots disabled; build plugins use the standard public repositories. Maven cache keys include the repository settings so changes to the Stage URL invalidate the cache. - -Loader's HDFS tests run separately, so other profiles do not wait for Hadoop. The Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate NameNode and DataNode containers on the host network. Tests run on Java 17 and use `localhost:8020`. Startup checks live DataNode registration and a real block write/read before testing; containers are removed afterward. Image pulling and readiness are bounded. Cold runners still need to pull the image. - -The immutable Server packages, `ci-plan` and successful `ci-test-results` artifacts +Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, +including Loader's separate HDFS jobs. Core module tests cover the locked candidate +and released Server 1.7 where configured; these runtime checks supplement affected-module +selection without adding unrelated modules. Their results remain advisory. + +The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked candidate +Server runs on Java 17 from a verified commit on `apache/hugegraph` master. +The SDK action, CI fixtures, image defaults and packaging verifier use the same +locked source commit. + +The released Server package and Hubble's candidate fixture are each built once and +shared, with independent services per job. Reuse verifies source, commit, JDK, build +inputs and archive checksum. Client, Loader and Tools candidate tests start the +archive already produced by their SDK bootstrap. SDK generation remains job-local +until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly +POM versions before installation, including the root parent, so dependency and license +metadata readers consume concrete coordinates without changing the locked source POMs. +The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. + +The shared Java initialization action requires Maven 3.9 or newer and supplies +explicit settings through `MAVEN_ARGS`, preserving existing arguments and user +settings. Dependencies are resolved from Central before ASF Stage, with Stage +snapshots disabled; build plugins use the standard public repositories. Maven cache +keys include the repository settings so changes to the Stage URL invalidate the cache. + +Loader validates its existing profiles against both configured Server baselines. The +Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate NameNode +and DataNode containers on the host network. Tests run on Java 17 and use +`localhost:8020`. Startup checks live DataNode registration and a real block +write/read before testing; containers are removed afterward. Image pulling and +readiness are bounded. Cold runners still need to pull the image. + +The immutable Server packages, `ci-plan` and `ci-test-results` diagnostic artifacts are retained for seven days from their upload. Partial reruns need the original plan and fixture within that window. After an artifact expires, rerun the whole workflow to rebuild its plan and fixtures; extending only the result report cannot @@ -69,35 +104,52 @@ selected consumers again. Source, type definitions, tests and CI configuration are never treated as docs. The gate checks the current workflow's actual selected job results and fixture producers. Its result report records the run, attempt and input metadata for diagnostics; it is not a reusable success receipt. -The planner determines affected consumers before resolving external Server inputs. Plans that require Server fixtures record both the historical release and the locked candidate source; Go-only module tests consume only the release fixture. Image-only and plain documentation plans need neither baseline. Unverifiable selection conservatively resolves both. A failed required baseline lookup fails planning; without a published plan, recovery requires a full workflow rerun. +The planner determines affected consumers before resolving external Server inputs. +Plans that require Server fixtures record the historical release and locked +candidate source; Go-only module tests consume only the release fixture. Image-only +and plain documentation plans need neither baseline. Unverifiable selection +conservatively resolves both. A failed selected baseline lookup fails planning; +without a published plan, recovery requires a full workflow rerun. PR planning records the event's head, base and source before querying live metadata. -A known mismatch with the checkout merge or current PR fails planning; expanding -coverage cannot make an outdated checkout current. After all selected jobs and -fixtures succeed, the gate rechecks the open PR's head, base and source using a -read-only API request. Changed inputs or unavailable metadata fail the gate. -Refresh the branch and start a new PR event after head or base movement: GitHub -reruns retain the original commit and event, so rerunning alone cannot refresh the -base. An unknown API or selection failure falls back to full coverage; a manual -full rerun can recover a transient failure while the recorded inputs remain current. - -CodeQL follows the module gate and retains its weekly scan. During migration, -`Analyze (java)` still runs on every PR, after a successful module gate. -The legacy alias is PR-only; documentation pushes do not force scanning or fail -because a deliberately unselected scan was skipped. Scan write permissions are -limited to the security job. After ASF branch protection actually -requires `check-license-header` and `affected-module-tests`, an administrator can -set `CI_OPTIMIZED_REQUIRED=true` to skip security scans for documentation-only -changes. This switch is disabled by default and does not alter branch protection. +A known mismatch with the checkout merge, current head or source fails planning. +The report records the actual tested base and merge; advancing master alone does +not invalidate a completed result, consistent with non-strict branch protection. +Unknown selection falls back to full validation. No cross-run success is reused. + +CodeQL starts independently on affected PR and push inputs, with its original +analysis identity and weekly full scan. Java, frontend and Python changes select +their supported scan language; shared/unknown build inputs keep all three languages. +Go-only and image-only changes do not start unrelated source scans. Manual and +weekly runs scan all three languages. Plain prose-only PRs skip compilation, +fixtures, images and scanning; native workflows retain lightweight selection jobs. +Packaged README files and shared build/configuration inputs are not plain prose. +The `CI_OPTIMIZED_REQUIRED` variable is no longer needed. + +The dependency inventory audit includes root `.mvn/**`, POMs, dependency manifests, +assembly/license inputs and staged Maven settings. It runs from the repository root +with the same root Maven configuration, separately from the required license check. + +For rollout, `.asf.yaml` requests only `check-license-header`. Confirm actual branch +protection has reconciled before relying on advisory cancellation. The existing +module summary remains available during migration. +`Analyze (java)` runs for Java or shared/unknown build inputs, and for manual and +weekly full scans; it is not emitted for every PR. Old required names must not +remain permanently Expected. A new PR head cancels older first attempts. Reruns use separate concurrency groups, so retrying an old commit cannot cancel the current head. Automatic retries run -failed **push** jobs at most twice, checking the repository branch head and +failed **push** license jobs only at most twice, checking the repository branch head and unchanged completed-failure run attempt both before and after the 180-second delay. Retry code comes from the trusted default branch. PR automatic retries and cross-run success reuse are deferred until execution evidence can be verified independently; manual workflow reruns remain available. +The parent-metadata Maven regression runs once in the selected shared fixture +for Java consumer validation, using a temporary Maven repository. Documentation +and Go-only PRs keep the lightweight planner; the manual fixture workflow also +runs this regression. + Validate policy and retry behavior locally: ```bash diff --git a/docs/java17-migration.md b/docs/java17-migration.md index d1a0397d9..497d509c6 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -4,9 +4,7 @@ Toolchain 1.8 requires Java 17 for the Java Client, ordinary Loader, Tools and H ## Build the locked candidate -TODO ([apache/hugegraph#3263](https://github.com/apache/hugegraph/pull/3263)): the organization repository, branch and commit below are temporary pre-merge validation inputs. After the Java 17 Server change merges, replace them with `apache/hugegraph` and a verified ASF commit, update the SDK action/verifier, Docker defaults, CI and examples together, and rerun dependency and runtime validation. Do not retain the temporary source after the merge. Prefer distinct, published SDK coordinates when they become available. - -The Java 17 server candidate retains the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. The locked source is `hugegraph/hugegraph`, branch `codex/cypher-minimal-compat`, commit `4c162f539b906fa06dd83228e3691b77fa5b77d7`. +The Java 17 server candidate retains the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. The locked source is `apache/hugegraph`, commit `d9abcd4317fb36128e7e4d209139ec7f3a28cdfc` from the upstream master branch after the Java 17 and TinkerPop 3.8 migration. Use an explicit JDK directory and check both Java and Maven. On systems where Java 17 is not registered, a system JDK selector can return another installed version. Checking the generated class version alone does not identify the JVM that built or tested it. @@ -21,10 +19,9 @@ mvn -version toolchain_root="$PWD" candidate_dir=$(mktemp -d /tmp/hugegraph-java17.XXXXXX) mkdir "$candidate_dir/m2" -# TODO(apache/hugegraph#3263): temporary pre-merge source; after merge use apache/hugegraph and a verified ASF SHA. -export SERVER_REPOSITORY=hugegraph/hugegraph -export SERVER_FETCH_REF=codex/cypher-minimal-compat -server_commit=4c162f539b906fa06dd83228e3691b77fa5b77d7 +export SERVER_REPOSITORY=apache/hugegraph +export SERVER_FETCH_REF=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc +server_commit=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc bash "$toolchain_root/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ "$server_commit" "$candidate_dir/server" "$candidate_dir/m2" @@ -32,7 +29,7 @@ mvn -Dmaven.repo.local="$candidate_dir/m2" clean install \ -DskipTests -Dmaven.javadoc.skip=true -ntp ``` -The SDK helper installs the complete locked server reactor without starting a server. It requires a new external source directory and a dedicated Maven repository with no existing HugeGraph artifacts. It records source and artifact provenance in `candidate-sdk-manifest.json` inside that repository. Reuse the same explicit `-Dmaven.repo.local` option for subsequent Toolchain validation. The manifest's source and JDK paths describe that build; use your own directories when rebuilding. The package command skips test execution; run the module test suites separately. These source-built packages are candidates, not an ASF release. +The SDK helper installs the locked Server distribution and its Maven reactor dependencies without starting a server. This includes every required SDK module and parent POM, but excludes unrelated Cluster tests and PD/Store service distributions. It requires a new external source directory and a dedicated Maven repository with no existing HugeGraph artifacts. It records source and artifact provenance in `candidate-sdk-manifest.json` inside that repository. Reuse the same explicit `-Dmaven.repo.local` option for subsequent Toolchain validation. The manifest's source and JDK paths describe that build; use your own directories when rebuilding. The package command skips test execution; run the module test suites separately. These source-built packages are candidates, not an ASF release. Distribution packaging also requires Python 3.9 or newer. Before each module archive is written, the existing SDK verifier checks the locked source, required POM/JAR hashes and local installation origins, then checks the distribution's manifest and actual SDK libraries. The combined archive rechecks all three module directories before moving them. Source checks lock Server SDK artifacts; Toolchain artifacts such as the Client, Loader and Hubble backend are outputs of this reactor and may change during `install`. Older manifests can retain their metadata, but those outputs are excluded from SDK source hashes. The original manifest is still retained unchanged and compared with each packaged copy. A missing manifest, a Central artifact with the same version, or a stale/changed packaged SDK library fails packaging. Ordinary `compile` and `test` do not require the candidate manifest; they may validate published dependencies without establishing candidate package identity. diff --git a/hugegraph-client/assembly/travis/install-candidate-sdk.sh b/hugegraph-client/assembly/travis/install-candidate-sdk.sh index 4d63d2d8c..55c6f3f96 100755 --- a/hugegraph-client/assembly/travis/install-candidate-sdk.sh +++ b/hugegraph-client/assembly/travis/install-candidate-sdk.sh @@ -36,15 +36,19 @@ export PATH="$JAVA_HOME/bin:$PATH" SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) SOURCE_DIR=$2 -# TODO(apache/hugegraph#3263): temporary pre-merge source; after merge use apache/hugegraph and a verified ASF SHA. -SERVER_REPOSITORY=${SERVER_REPOSITORY:-hugegraph/hugegraph} +SERVER_REPOSITORY=${SERVER_REPOSITORY:-apache/hugegraph} SERVER_FETCH_REF=${SERVER_FETCH_REF:-$1} export SERVER_REPOSITORY SERVER_FETCH_REF CANDIDATE_REPO=$3 bash "$SCRIPT_DIR/checkout-server.sh" "$1" "$SOURCE_DIR" -# Install the entire locked reactor so Common, Core, PD and Store share one source. +# Install the Server distribution's dependency reactor so all SDK modules share one source. # The fixture installers retain package-only semantics and use another repository. -(cd "$SOURCE_DIR" && mvn "-Dmaven.repo.local=$CANDIDATE_REPO" install \ +# Flatten the root too: license metadata rebuilds installed parent models, whose +# raw ${revision} otherwise becomes an invalid artifact request outside the reactor. +(cd "$SOURCE_DIR" && mvn "-Dmaven.repo.local=$CANDIDATE_REPO" \ + org.codehaus.mojo:flatten-maven-plugin:1.2.7:flatten install \ + -pl hugegraph-server/hugegraph-dist -am \ + -Dflatten.mode=resolveCiFriendliesOnly -DupdatePomFile=true \ -DskipTests -Dmaven.javadoc.skip=true -ntp) ACTUAL_COMMIT=$(git -C "$SOURCE_DIR" rev-parse HEAD) python3 - "$SOURCE_DIR" "$CANDIDATE_REPO" "${SERVER_REPOSITORY:-apache/hugegraph}" \ diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs index 66838e9a8..8fbd54baa 100644 --- a/hugegraph-client/assembly/travis/install-server.test.cjs +++ b/hugegraph-client/assembly/travis/install-server.test.cjs @@ -404,7 +404,10 @@ test('candidate SDK installs the verified reactor into an explicit repository an const result = run('checkout'); assert.equal(result.status, 0, result.stderr); assert.deepEqual(readFileSync(env.BUILD_ARGS, 'utf8').trim().split('\n'), - [`-Dmaven.repo.local=${repo}`, 'install', '-DskipTests', '-Dmaven.javadoc.skip=true', '-ntp']); + [`-Dmaven.repo.local=${repo}`, 'org.codehaus.mojo:flatten-maven-plugin:1.2.7:flatten', 'install', + '-pl', 'hugegraph-server/hugegraph-dist', '-am', + '-Dflatten.mode=resolveCiFriendliesOnly', '-DupdatePomFile=true', + '-DskipTests', '-Dmaven.javadoc.skip=true', '-ntp']); const manifest = JSON.parse(readFileSync(join(repo, 'candidate-sdk-manifest.json'), 'utf8')); assert.equal(manifest.repository, 'example/server'); assert.equal(manifest.commit, sha); diff --git a/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-dist/scripts/dependency/known-dependencies.txt index ea0cfbc83..1e8dce0c0 100644 --- a/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -337,7 +337,7 @@ jna-5.12.1.jar joda-time-2.10.8.jar joni-2.1.11.jar jpam-1.1.jar -jraft-core-1.3.11.jar +jraft-core-1.3.14.jar jsch-0.1.55.jar json-1.8.jar json-io-2.5.1.jar diff --git a/hugegraph-hubble/Dockerfile b/hugegraph-hubble/Dockerfile index 2e004f1af..c498e5677 100644 --- a/hugegraph-hubble/Dockerfile +++ b/hugegraph-hubble/Dockerfile @@ -36,13 +36,12 @@ COPY hugegraph-client/assembly/travis/install-candidate-sdk.sh \ COPY .github/scripts/verify_candidate_image_sdk.py /pkg/.github/scripts/ # Direct builds bootstrap the same locked SDK; CI can supply a verified builder. -# TODO(apache/hugegraph#3263): after merge replace this temporary source with apache/hugegraph and an ASF SHA. RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ - SERVER_REPOSITORY=hugegraph/hugegraph \ - SERVER_FETCH_REF=codex/cypher-minimal-compat \ + SERVER_REPOSITORY=apache/hugegraph \ + SERVER_FETCH_REF=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - 4c162f539b906fa06dd83228e3691b77fa5b77d7 /opt/candidate-source /opt/candidate-m2; \ + d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/test_configure_gremlin_fixture.py b/hugegraph-hubble/hubble-dist/assembly/travis/test_configure_gremlin_fixture.py index a9c635a4b..2e351af42 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/test_configure_gremlin_fixture.py +++ b/hugegraph-hubble/hubble-dist/assembly/travis/test_configure_gremlin_fixture.py @@ -76,10 +76,9 @@ def test_archive_commit_and_hash_bind_the_supplied_payload(self): output = root / "output" output.mkdir() commit = "1" * 40 - # TODO(apache/hugegraph#3263): align this temporary source fixture with ASF after the merge. env = dict(os.environ, SERVER_ARCHIVE=str(archive), SERVER_ARCHIVE_COMMIT=commit, SERVER_ARCHIVE_SHA256=hashlib.sha256(archive.read_bytes()).hexdigest(), - SERVER_REPOSITORY="hugegraph/hugegraph") + SERVER_REPOSITORY="apache/hugegraph") result = subprocess.run(["bash", str((HERE / "download-hugegraph.sh").resolve()), commit], cwd=output, env=env, capture_output=True) self.assertEqual(0, result.returncode, result.stderr.decode()) diff --git a/hugegraph-loader/Dockerfile b/hugegraph-loader/Dockerfile index 6cc99be19..e693a6b79 100644 --- a/hugegraph-loader/Dockerfile +++ b/hugegraph-loader/Dockerfile @@ -32,13 +32,12 @@ COPY hugegraph-client/assembly/travis/install-candidate-sdk.sh \ COPY .github/scripts/verify_candidate_image_sdk.py /pkg/.github/scripts/ # Direct builds bootstrap the same locked SDK; CI can supply a verified builder. -# TODO(apache/hugegraph#3263): after merge replace this temporary source with apache/hugegraph and an ASF SHA. RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ - SERVER_REPOSITORY=hugegraph/hugegraph \ - SERVER_FETCH_REF=codex/cypher-minimal-compat \ + SERVER_REPOSITORY=apache/hugegraph \ + SERVER_FETCH_REF=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - 4c162f539b906fa06dd83228e3691b77fa5b77d7 /opt/candidate-source /opt/candidate-m2; \ + d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 From 7e6310439243d4da83b541f844efbbf7444c09fa Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 18:52:14 +0800 Subject: [PATCH 26/38] docs(ci): keep paragraphs on single lines - retain the merged CI policy wording - avoid early paragraph wrapping - preserve tables and code blocks --- docs/ci.md | 153 +++++++++++------------------------------------------ 1 file changed, 32 insertions(+), 121 deletions(-) diff --git a/docs/ci.md b/docs/ci.md index d635c6232..1087bbe7b 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,8 +1,6 @@ # Continuous integration -Only `check-license-header` (headers and RAT) is required for merging. Execution -selection remains conservative: changed modules and their consumers run, but their -results are advisory. Maintainers decide whether to rerun or request changes. +Only `check-license-header` (headers and RAT) is required for merging. Execution selection remains conservative: changed modules and their consumers run, but their results are advisory. Maintainers decide whether to rerun or request changes. The independent workflows are: @@ -12,11 +10,7 @@ The independent workflows are: | Module validation | Consumers, images, fixtures and dependency inventory audit | No | Can be cancelled without cancelling license checks or CodeQL | | CodeQL | Affected-input scans and dependency review; weekly full scan | No | Independent of module tests | -`affected-module-tests` remains a truthful advisory summary under its existing -name for migration. It fails when a selected test, image or fixture fails, is -cancelled or unexpectedly skipped; it does not make these checks required. -Plan and result summaries show specific checks, triggering paths, required roles -and actual results. Native job checks provide the detailed logs. +`affected-module-tests` remains a truthful advisory summary under its existing name for migration. It fails when a selected test, image or fixture fails, is cancelled or unexpectedly skipped; it does not make these checks required. Plan and result summaries show specific checks, triggering paths, required roles and actual results. Native job checks provide the detailed logs. | Change | Tests | | --- | --- | @@ -27,8 +21,7 @@ and actual results. Native job checks provide the detailed logs. | Shared build inputs or unrecognized paths | All modules | | One workflow | Its tests and shared dependencies | -Image selection has its own matrix and does not follow the Client → Loader → -Hubble module-test dependency closure: +Image selection has its own matrix and does not follow the Client → Loader → Hubble module-test dependency closure: | Image input change | Images built and started | | --- | --- | @@ -39,117 +32,35 @@ Hubble module-test dependency closure: | Root/Client/reactor POMs, `.mvn`, `.dockerignore`, shared release docs or unknown shared inputs | Loader and Hubble | | Ordinary Java/frontend application source or tests | None; the module behavior tests above still run | -Dockerfile-only changes run the image check without building Server fixtures. -Hubble's Dockerfile builds Client and Loader before Hubble, so Loader packaging -changes also select Hubble. Packaged README changes select images; the existing -nonempty Hubble README content contract still requires fresh validation if the -file is missing, empty or has an invalid mode. Every selected module and image -runs in the current workflow; successful results from another run are not reused. - -Each image job builds the checkout's actual Dockerfile, records its image ID and -starts a local run-scoped tag with pulling disabled. The container's image ID must -match the build. Loader directly executes `./bin/hugegraph-loader.sh --help`, -including its executable permission and shebang; its default idle -container alone cannot pass. Hubble must remain running and return a valid `/about` -JSON response with the application name and version. Its public root page must -contain the React root and serve nonempty local JavaScript bundles and referenced -stylesheets with the correct content types, rejecting the HTML fallback for missing -assets. These checks validate image build, packaging, startup and public entry -points; Loader data ingestion and Hubble browser workflows remain covered by -module tests. They do not publish an image. - -Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, -including Loader's separate HDFS jobs. Core module tests cover the locked candidate -and released Server 1.7 where configured; these runtime checks supplement affected-module -selection without adding unrelated modules. Their results remain advisory. - -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF -master Server uses Java 17 and TinkerPop 3.8.1. CI resolves the six-character -source selector `d9abcd` to a complete immutable commit, so later master updates -do not change the build inputs. The SDK action, CI fixtures, image defaults and -packaging verifier use the same locked source commit. - -The released Server package and Hubble's candidate fixture are each built once and -shared, with independent services per job. Reuse verifies source, commit, JDK, build -inputs and archive checksum. Client, Loader and Tools candidate tests start the -archive already produced by their SDK bootstrap. SDK generation remains job-local -until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly -POM versions before installation, including the root parent, so dependency and license -metadata readers consume concrete coordinates without changing the locked source POMs. -The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. - -The shared Java initialization action requires Maven 3.9 or newer and supplies -explicit settings through `MAVEN_ARGS`, preserving existing arguments and user -settings. Dependencies are resolved from Central before ASF Stage, with Stage -snapshots disabled; build plugins use the standard public repositories. Maven cache -keys include the repository settings so changes to the Stage URL invalidate the cache. - -Loader validates its existing profiles against both configured Server baselines. The -Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate NameNode -and DataNode containers on the host network. Tests run on Java 17 and use -`localhost:8020`. Startup checks live DataNode registration and a real block -write/read before testing; containers are removed afterward. Image pulling and -readiness are bounded. Cold runners still need to pull the image. - -The immutable Server packages, `ci-plan` and `ci-test-results` diagnostic artifacts -are retained for seven days from their upload. Partial reruns need the original -plan and fixture within that window. After an artifact expires, rerun the whole -workflow to rebuild its plan and fixtures; extending only the result report cannot -restore an expired package. Test reports and coverage artifacts keep their -existing retention settings. - -Documentation paths use an explicit allowlist. A PR containing only plain prose -and static documentation assets has no module tests. Selection uses the cumulative -PR diff, so a documentation update to a PR that also changes source runs its -selected consumers again. Source, type definitions, tests and CI configuration -are never treated as docs. The gate checks the current workflow's actual selected -job results and fixture producers. Its result report records the run, attempt and -input metadata for diagnostics; it is not a reusable success receipt. -The planner determines affected consumers before resolving external Server inputs. -Plans that require Server fixtures record the historical release and locked -candidate source; Go-only module tests consume only the release fixture. Image-only -and plain documentation plans need neither baseline. Unverifiable selection -conservatively resolves both. A failed selected baseline lookup fails planning; -without a published plan, recovery requires a full workflow rerun. - -PR planning records the event's head, base and source before querying live metadata. -A known mismatch with the checkout merge, current head or source fails planning. -The report records the actual tested base and merge; advancing master alone does -not invalidate a completed result, consistent with non-strict branch protection. -Unknown selection falls back to full validation. No cross-run success is reused. - -CodeQL starts independently on affected PR and push inputs, with its original -analysis identity and weekly full scan. Java, frontend and Python changes select -their supported scan language; shared/unknown build inputs keep all three languages. -Go-only and image-only changes do not start unrelated source scans. Manual and -weekly runs scan all three languages. Plain prose-only PRs skip compilation, -fixtures, images and scanning; native workflows retain lightweight selection jobs. -Packaged README files and shared build/configuration inputs are not plain prose. -The `CI_OPTIMIZED_REQUIRED` variable is no longer needed. - -The dependency inventory audit includes root `.mvn/**`, POMs, dependency manifests, -assembly/license inputs and staged Maven settings. It runs from the repository root -with the same root Maven configuration, separately from the required license check. - -For rollout, `.asf.yaml` requests only `check-license-header`. Confirm actual branch -protection has reconciled before relying on advisory cancellation. The existing -module summary remains available during migration. -`Analyze (java)` runs for Java or shared/unknown build inputs, and for manual and -weekly full scans; it is not emitted for every PR. Old required names must not -remain permanently Expected. - -A new PR head cancels older first attempts. Reruns use separate concurrency groups, -so retrying an old commit cannot cancel the current head. Automatic retries run -failed **push** license jobs only at most twice, checking the repository branch head and -unchanged completed-failure run attempt both before and after the 180-second delay. -Retry code comes from the trusted default branch. PR automatic retries and -cross-run success reuse are deferred until execution evidence can be verified -independently; manual workflow reruns remain available. - -The parent-metadata Maven regression runs once in the selected shared fixture -for Java consumer validation, using a temporary Maven repository. Documentation -and Go-only PRs keep the lightweight planner; the manual fixture workflow also -runs this regression. +Dockerfile-only changes run the image check without building Server fixtures. Hubble's Dockerfile builds Client and Loader before Hubble, so Loader packaging changes also select Hubble. Packaged README changes select images; the existing nonempty Hubble README content contract still requires fresh validation if the file is missing, empty or has an invalid mode. Every selected module and image runs in the current workflow; successful results from another run are not reused. + +Each image job builds the checkout's actual Dockerfile, records its image ID and starts a local run-scoped tag with pulling disabled. The container's image ID must match the build. Loader directly executes `./bin/hugegraph-loader.sh --help`, including its executable permission and shebang; its default idle container alone cannot pass. Hubble must remain running and return a valid `/about` JSON response with the application name and version. Its public root page must contain the React root and serve nonempty local JavaScript bundles and referenced stylesheets with the correct content types, rejecting the HTML fallback for missing assets. These checks validate image build, packaging, startup and public entry points; Loader data ingestion and Hubble browser workflows remain covered by module tests. They do not publish an image. + +Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement affected-module selection without adding unrelated modules. Their results remain advisory. + +The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. CI resolves the six-character source selector `d9abcd` to a complete immutable commit, so later master updates do not change the build inputs. The SDK action, CI fixtures, image defaults and packaging verifier use the same locked source commit. + +The released Server package and Hubble's candidate fixture are each built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly POM versions before installation, including the root parent, so dependency and license metadata readers consume concrete coordinates without changing the locked source POMs. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. + +The shared Java initialization action requires Maven 3.9 or newer and supplies explicit settings through `MAVEN_ARGS`, preserving existing arguments and user settings. Dependencies are resolved from Central before ASF Stage, with Stage snapshots disabled; build plugins use the standard public repositories. Maven cache keys include the repository settings so changes to the Stage URL invalidate the cache. + +Loader validates its existing profiles against both configured Server baselines. The Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate NameNode and DataNode containers on the host network. Tests run on Java 17 and use `localhost:8020`. Startup checks live DataNode registration and a real block write/read before testing; containers are removed afterward. Image pulling and readiness are bounded. Cold runners still need to pull the image. + +The immutable Server packages, `ci-plan` and `ci-test-results` diagnostic artifacts are retained for seven days from their upload. Partial reruns need the original plan and fixture within that window. After an artifact expires, rerun the whole workflow to rebuild its plan and fixtures; extending only the result report cannot restore an expired package. Test reports and coverage artifacts keep their existing retention settings. + +Documentation paths use an explicit allowlist. A PR containing only plain prose and static documentation assets has no module tests. Selection uses the cumulative PR diff, so a documentation update to a PR that also changes source runs its selected consumers again. Source, type definitions, tests and CI configuration are never treated as docs. The gate checks the current workflow's actual selected job results and fixture producers. Its result report records the run, attempt and input metadata for diagnostics; it is not a reusable success receipt. The planner determines affected consumers before resolving external Server inputs. Plans that require Server fixtures record the historical release and locked candidate source; Go-only module tests consume only the release fixture. Image-only and plain documentation plans need neither baseline. Unverifiable selection conservatively resolves both. A failed selected baseline lookup fails planning; without a published plan, recovery requires a full workflow rerun. + +PR planning records the event's head, base and source before querying live metadata. A known mismatch with the checkout merge, current head or source fails planning. The report records the actual tested base and merge; advancing master alone does not invalidate a completed result, consistent with non-strict branch protection. Unknown selection falls back to full validation. No cross-run success is reused. + +CodeQL starts independently on affected PR and push inputs, with its original analysis identity and weekly full scan. Java, frontend and Python changes select their supported scan language; shared/unknown build inputs keep all three languages. Go-only and image-only changes do not start unrelated source scans. Manual and weekly runs scan all three languages. Plain prose-only PRs skip compilation, fixtures, images and scanning; native workflows retain lightweight selection jobs. Packaged README files and shared build/configuration inputs are not plain prose. The `CI_OPTIMIZED_REQUIRED` variable is no longer needed. + +The dependency inventory audit includes root `.mvn/**`, POMs, dependency manifests, assembly/license inputs and staged Maven settings. It runs from the repository root with the same root Maven configuration, separately from the required license check. + +For rollout, `.asf.yaml` requests only `check-license-header`. Confirm actual branch protection has reconciled before relying on advisory cancellation. The existing module summary remains available during migration. `Analyze (java)` runs for Java or shared/unknown build inputs, and for manual and weekly full scans; it is not emitted for every PR. Old required names must not remain permanently Expected. + +A new PR head cancels older first attempts. Reruns use separate concurrency groups, so retrying an old commit cannot cancel the current head. Automatic retries run failed **push** license jobs only at most twice, checking the repository branch head and unchanged completed-failure run attempt both before and after the 180-second delay. Retry code comes from the trusted default branch. PR automatic retries and cross-run success reuse are deferred until execution evidence can be verified independently; manual workflow reruns remain available. + +The parent-metadata Maven regression runs once in the selected shared fixture for Java consumer validation, using a temporary Maven repository. Documentation and Go-only PRs keep the lightweight planner; the manual fixture workflow also runs this regression. Validate policy and retry behavior locally: From 7286290bf6313ca29452071150c8c67d17c4d9f6 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 22:25:11 +0800 Subject: [PATCH 27/38] fix: restore full source commit locks - pin CI, SDK bootstrap and image builds to full source identities - remove abbreviated commit API expansion from checkout and packaging - retain verified full commit context for upstream canary validation - show short identities in docs and read the execution lock locally --- .../actions/get-hugegraph-commit/action.yml | 2 +- .../actions/get-hugegraph-commit/resolve.cjs | 22 +++------- .../get-hugegraph-commit/resolve.test.cjs | 25 +++-------- .../actions/setup-candidate-sdk/action.yml | 4 +- .github/scripts/test_candidate_sdk.py | 17 +++----- .github/scripts/verify_candidate_image_sdk.py | 23 +---------- .github/workflows/ci.yml | 6 +-- docs/ci.md | 2 +- docs/java17-migration.md | 8 +++- .../assembly/travis/checkout-server.sh | 41 +------------------ .../assembly/travis/install-server.test.cjs | 19 --------- hugegraph-hubble/Dockerfile | 2 +- hugegraph-loader/Dockerfile | 2 +- 13 files changed, 36 insertions(+), 137 deletions(-) diff --git a/.github/actions/get-hugegraph-commit/action.yml b/.github/actions/get-hugegraph-commit/action.yml index df3d4c0a1..60152b0d1 100644 --- a/.github/actions/get-hugegraph-commit/action.yml +++ b/.github/actions/get-hugegraph-commit/action.yml @@ -13,7 +13,7 @@ inputs: description: 'Branch, tag or commit; mutually exclusive with server-pr-number' default: '' expected-commit: - description: 'Optional six-character or full commit SHA; fail if the selected ref has moved' + description: 'Optional full commit SHA; fail if the selected ref has moved' default: '' outputs: diff --git a/.github/actions/get-hugegraph-commit/resolve.cjs b/.github/actions/get-hugegraph-commit/resolve.cjs index 556f986d5..ae0ea897b 100644 --- a/.github/actions/get-hugegraph-commit/resolve.cjs +++ b/.github/actions/get-hugegraph-commit/resolve.cjs @@ -19,8 +19,8 @@ module.exports = async function resolve(github, { repository, ref = '', pr = '', if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(repository)) { throw new Error('Invalid server repository'); } - if (expectedCommit && !/^(?:[0-9a-f]{6}|[0-9a-f]{40})$/i.test(expectedCommit)) { - throw new Error('Expected commit must be a six-character or full SHA'); + if (expectedCommit && !/^[0-9a-f]{40}$/i.test(expectedCommit)) { + throw new Error('Expected commit must be a full SHA'); } if (ref && pr) throw new Error('Select either ref or server-pr-number'); const [owner, repo] = repository.split('/'); @@ -46,23 +46,13 @@ module.exports = async function resolve(github, { repository, ref = '', pr = '', } if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error('Server API returned an invalid commit SHA'); if (expectedCommit) { - let expectedSha = expectedCommit; - if (expectedCommit.length === 6) { - // Resolve independently: a matching prefix alone does not establish identity. - const { data: expected } = await github.rest.repos.getCommit({ owner, repo, ref: expectedCommit }); - expectedSha = expected.sha; - if (!/^[0-9a-f]{40}$/i.test(expectedSha) || - !expectedSha.toLowerCase().startsWith(expectedCommit.toLowerCase())) { - throw new Error('Server API returned an invalid expected commit SHA'); - } - } - if (sha.toLowerCase() !== expectedSha.toLowerCase()) { + if (sha.toLowerCase() !== expectedCommit.toLowerCase()) { throw new Error(`Server ref moved: expected ${expectedCommit.slice(0, 6)}, resolved ${sha.slice(0, 6)}`); } } - if (/^(?:[0-9a-f]{6}|[0-9a-f]{40})$/i.test(ref)) { - if (!sha.toLowerCase().startsWith(ref.toLowerCase())) { - throw new Error('Server API returned a commit outside the selected prefix'); + if (/^[0-9a-f]{40}$/i.test(ref)) { + if (sha.toLowerCase() !== ref.toLowerCase()) { + throw new Error('Server API returned a commit different from the selected SHA'); } fetchRef = sha; } diff --git a/.github/actions/get-hugegraph-commit/resolve.test.cjs b/.github/actions/get-hugegraph-commit/resolve.test.cjs index aa1383cd7..62ceb0b9c 100644 --- a/.github/actions/get-hugegraph-commit/resolve.test.cjs +++ b/.github/actions/get-hugegraph-commit/resolve.test.cjs @@ -57,7 +57,7 @@ test('resolves PR head from the selected repository', async () => { test('rejects invalid or ambiguous selectors before calling GitHub', async () => { for (const input of [{ repository: '../repo' }, { ref: 'master', pr: '42' }, - { pr: '1.5' }, { pr: '0' }, { expectedCommit: 'short' }]) { + { pr: '1.5' }, { pr: '0' }, { expectedCommit: 'short' }, { expectedCommit: sha.slice(0, 6) }]) { const { github, calls } = client(); await assert.rejects(resolve(github, { repository: 'apache/hugegraph', ...input })); assert.equal(calls.length, 0); @@ -72,26 +72,11 @@ test('fails closed for a moved branch or API error', async () => { await assert.rejects(resolve(github, { repository: 'example/server', ref: 'missing' }), /Not Found/); }); -test('resolves an abbreviated commit to a complete fetch identity', async () => { +test('verifies a full commit selector and fetch identity', async () => { const { github, calls } = client(); - const short = sha.slice(0, 6); assert.deepEqual(await resolve(github, { repository: 'apache/hugegraph', - ref: short, expectedCommit: short }), { sha, fetchRef: sha }); - assert.ok(calls.every(([, args]) => args.ref === short)); -}); - -test('checks complete identity even when resolved commits share the short prefix', async () => { - const { github } = client(); - github.rest.repos.getCommit = async ({ ref }) => ({ data: { - sha: ref === 'master' ? sha : sha.slice(0, 6) + 'b'.repeat(34) - } }); - await assert.rejects(resolve(github, { repository: 'apache/hugegraph', - ref: 'master', expectedCommit: sha.slice(0, 6) }), /Server ref moved/); -}); - -test('rejects abbreviated commits that GitHub cannot resolve', async () => { - const { github } = client(); - github.rest.repos.getCommit = async () => { throw new Error('Ambiguous commit'); }; + ref: sha, expectedCommit: sha }), { sha, fetchRef: sha }); + assert.deepEqual(calls, [['commit', { owner: 'apache', repo: 'hugegraph', ref: sha }]]); await assert.rejects(resolve(github, { repository: 'apache/hugegraph', - ref: sha.slice(0, 6), expectedCommit: sha.slice(0, 6) }), /Ambiguous commit/); + ref: 'b'.repeat(40) }), /different from the selected SHA/); }); diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index 552c3afe8..7bef989fb 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -7,10 +7,10 @@ inputs: default: 'apache/hugegraph' ref: description: 'Immutable candidate source commit' - default: 'd9abcd' + default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' expected-commit: description: 'Locked candidate source commit' - default: 'd9abcd' + default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' outputs: repository: diff --git a/.github/scripts/test_candidate_sdk.py b/.github/scripts/test_candidate_sdk.py index 95a025c51..1621defeb 100644 --- a/.github/scripts/test_candidate_sdk.py +++ b/.github/scripts/test_candidate_sdk.py @@ -108,7 +108,7 @@ def run(*arguments): class CandidateDistributionTest(unittest.TestCase): def setUp(self): - self.source_commit = sdk.COMMIT + "a" * (40 - len(sdk.COMMIT)) + self.source_commit = sdk.COMMIT context = patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": self.source_commit}) context.start() self.addCleanup(context.stop) @@ -267,14 +267,14 @@ def test_reactor_installs_do_not_change_the_locked_sdk_contract(self): with self.assertRaisesRegex(RuntimeError, "SDK artifact hash mismatch"): sdk.validate_distribution(repository, directory, "hubble") - def test_short_source_context_preserves_full_identity_checks(self): + def test_source_context_preserves_full_identity_checks(self): with tempfile.TemporaryDirectory() as temporary: repository, manifest = self.fixture(Path(temporary)) - manifest["commit"] = sdk.COMMIT + "b" * (40 - len(sdk.COMMIT)) + manifest["commit"] = sdk.COMMIT[:6] + "b" * 34 self.write_manifest(repository, manifest) with self.assertRaisesRegex(RuntimeError, "SDK source does not match"): sdk.validate_sdk(repository) - with patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": sdk.COMMIT}): + with patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": sdk.COMMIT[:6]}): with self.assertRaisesRegex(RuntimeError, "Invalid candidate SDK manifest"): sdk.validate_sdk(repository) @@ -289,16 +289,11 @@ def test_explicit_resolved_context_supports_latest_master_provenance(self): with self.assertRaisesRegex(RuntimeError, "SDK source does not match"): sdk.validate_sdk(repository) - def test_standalone_packaging_resolves_the_short_lock(self): + def test_standalone_packaging_uses_the_local_lock(self): with tempfile.TemporaryDirectory() as temporary: repository, _ = self.fixture(Path(temporary)) - with patch.dict(os.environ, {}, clear=True), patch.object(sdk, "_resolve_source_commit") as resolve: - resolve.return_value = self.source_commit + with patch.dict(os.environ, {}, clear=True): sdk.validate_sdk(repository) - resolve.assert_called_once_with("apache/hugegraph", sdk.COMMIT) - resolve.side_effect = OSError("ambiguous or inaccessible") - with self.assertRaisesRegex(RuntimeError, "Invalid candidate SDK manifest"): - sdk.validate_sdk(repository) def test_candidate_manifest_requires_the_supported_build_jdk(self): with tempfile.TemporaryDirectory() as temporary: diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index 2a6deab28..1ee62069e 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -22,12 +22,10 @@ import json import os import re -from functools import lru_cache -import urllib.request from pathlib import Path REPOSITORY = "apache/hugegraph" -COMMIT = "d9abcd" +COMMIT = "d9abcd4317fb36128e7e4d209139ec7f3a28cdfc" REQUIRED_MODULES = { "pom.xml", "hugegraph-commons/pom.xml", "hugegraph-server/pom.xml", "hugegraph-pd/pom.xml", "hugegraph-store/pom.xml", @@ -55,26 +53,9 @@ def digest(path): return checksum.hexdigest() - -@lru_cache(maxsize=1) -def _resolve_source_commit(repository, commit): - headers = {"Accept": "application/vnd.github+json", "User-Agent": "hugegraph-toolchain-ci"} - token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") - if token: - headers["Authorization"] = "Bearer " + token - request = urllib.request.Request(f"https://api.github.com/repos/{repository}/commits/{commit}", headers=headers) - with urllib.request.urlopen(request, timeout=30) as response: - return json.load(response)["sha"] - - def source_commit(): # The SDK action exports the full identity already verified by the source resolver. - # Standalone packaging resolves the configured short commit through GitHub instead. - expected = os.environ.get("CANDIDATE_SOURCE_COMMIT") - if expected is None: - expected = _resolve_source_commit(REPOSITORY, COMMIT) - if not isinstance(expected, str) or not expected.startswith(COMMIT): - raise ValueError("Invalid resolved Server source identity") + expected = os.environ.get("CANDIDATE_SOURCE_COMMIT", COMMIT) if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{40}", expected): raise ValueError("Invalid resolved Server source identity") return expected diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1488f2dc9..3c4b38d6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,15 +79,15 @@ jobs: with: repository: apache/hugegraph ref: refs/tags/1.7.0 - expected-commit: b12425 + expected-commit: b12425c2032bf0d21a97b8221f42a18055c2982f - name: Resolve locked Java17 candidate baseline id: hubble-baseline if: steps.scope.outputs.needsFixture == 'true' uses: ./.github/actions/get-hugegraph-commit with: repository: apache/hugegraph - ref: d9abcd - expected-commit: d9abcd + ref: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc + expected-commit: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc - name: Record external test inputs env: NEEDS_SERVER: ${{ steps.scope.outputs.needsFixture }} diff --git a/docs/ci.md b/docs/ci.md index 1087bbe7b..3479638b7 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -38,7 +38,7 @@ Each image job builds the checkout's actual Dockerfile, records its image ID and Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement affected-module selection without adding unrelated modules. Their results remain advisory. -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. CI resolves the six-character source selector `d9abcd` to a complete immutable commit, so later master updates do not change the build inputs. The SDK action, CI fixtures, image defaults and packaging verifier use the same locked source commit. +The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. CI locks the complete immutable source commit (shown as `d9abcd` here), so later master updates do not change the build inputs. The SDK action, CI fixtures, image defaults and packaging verifier use the same locked source commit. The released Server package and Hubble's candidate fixture are each built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly POM versions before installation, including the root parent, so dependency and license metadata readers consume concrete coordinates without changing the locked source POMs. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. diff --git a/docs/java17-migration.md b/docs/java17-migration.md index 3b89ddb1d..b07f3b36f 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -6,7 +6,7 @@ Toolchain builds, tests and applications use Java 17. The released Server 1.7 co ## Build the locked candidate -The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses the verified `apache/hugegraph` master baseline `d9abcd`, resolving the six-character commit through GitHub and fetching the complete immutable identity instead of the moving branch. Short prefixes that are ambiguous or inaccessible fail before fetching. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. +The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses the verified `apache/hugegraph` master baseline (shown as `d9abcd` here), storing and fetching its complete immutable commit identity. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. Use an explicit JDK directory and check both Java and Maven. On systems where Java 17 is not registered, a system JDK selector can return another installed version. Checking the generated class version alone does not identify the JVM that built or tested it. @@ -22,7 +22,11 @@ toolchain_root="$PWD" candidate_dir=$(mktemp -d /tmp/hugegraph-java17.XXXXXX) mkdir "$candidate_dir/m2" export SERVER_REPOSITORY=apache/hugegraph -server_commit=d9abcd +server_commit=$(python3 - <<'PY_COMMIT' +import runpy +print(runpy.run_path(".github/scripts/verify_candidate_image_sdk.py")["COMMIT"]) +PY_COMMIT +) bash "$toolchain_root/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ "$server_commit" "$candidate_dir/server" "$candidate_dir/m2" diff --git a/hugegraph-client/assembly/travis/checkout-server.sh b/hugegraph-client/assembly/travis/checkout-server.sh index a4314f38e..827ac155f 100644 --- a/hugegraph-client/assembly/travis/checkout-server.sh +++ b/hugegraph-client/assembly/travis/checkout-server.sh @@ -26,49 +26,12 @@ COMMIT_ID=$1 CHECKOUT_DIR=$2 SERVER_REPOSITORY=${SERVER_REPOSITORY:-apache/hugegraph} SERVER_FETCH_REF=${SERVER_FETCH_REF:-$COMMIT_ID} -if [[ ! "$COMMIT_ID" =~ ^([0-9a-fA-F]{6}|[0-9a-fA-F]{40})$ ]] || +if [[ ! "$COMMIT_ID" =~ ^[0-9a-fA-F]{40}$ ]] || [[ ! "$SERVER_REPOSITORY" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*/[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then - echo "Expected a six-character or full commit SHA and GitHub owner/repository" >&2 + echo "Expected a full commit SHA and GitHub owner/repository" >&2 exit 1 fi -if [[ ${#COMMIT_ID} -eq 6 ]]; then - SHORT_COMMIT=$COMMIT_ID - COMMIT_ID=$(python3 - "$SERVER_REPOSITORY" "$SHORT_COMMIT" <<'PY_RESOLVE' -import json -import os -import re -import sys -import urllib.error -import urllib.request - -repository, short = sys.argv[1:] -headers = {'Accept': 'application/vnd.github+json', 'User-Agent': 'hugegraph-toolchain-ci'} -token = os.environ.get('GITHUB_TOKEN') or os.environ.get('GH_TOKEN') -if token: - headers['Authorization'] = 'Bearer ' + token -request = urllib.request.Request(f'https://api.github.com/repos/{repository}/commits/{short}', headers=headers) -try: - with urllib.request.urlopen(request, timeout=30) as response: - sha = json.load(response)['sha'] - if not isinstance(sha, str) or not re.fullmatch(r'[0-9a-fA-F]{40}', sha) or not sha.lower().startswith(short.lower()): - raise ValueError('Invalid commit identity') -except (OSError, ValueError, KeyError, TypeError): - raise SystemExit('Unable to resolve the abbreviated Server commit') from None -print(sha.lower()) -PY_RESOLVE - ) -fi -# Git transport requires a complete object ID, not an abbreviated commit prefix. -if [[ "$SERVER_FETCH_REF" =~ ^[0-9a-fA-F]{6}$ ]]; then - FETCH_PREFIX=$(printf '%s' "$SERVER_FETCH_REF" | tr '[:upper:]' '[:lower:]') - if [[ "$(printf '%s' "$COMMIT_ID" | tr '[:upper:]' '[:lower:]')" != "$FETCH_PREFIX"* ]]; then - echo "Server checkout mismatch: selected commit differs from fetch prefix" >&2 - exit 1 - fi - SERVER_FETCH_REF=$COMMIT_ID -fi - # Fetch a reachable ref, then verify it against the resolved SHA before building. # A moving branch or PR must fail closed rather than silently changing the baseline. [[ ! -e "$CHECKOUT_DIR" ]] || { echo "Checkout directory already exists" >&2; exit 1; } diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs index 9b74560f2..7c3fad105 100644 --- a/hugegraph-client/assembly/travis/install-server.test.cjs +++ b/hugegraph-client/assembly/travis/install-server.test.cjs @@ -507,22 +507,3 @@ for (const scenario of [ assert.equal(readdirSync(root).some(name => name.startsWith('hugegraph-servers.')), false); }); } - -test('checkout resolves short source selectors before Git transport', t => { - const root = temp(t); - const bin = join(root, 'bin'); - mkdirSync(bin); - const full = 'a'.repeat(40); - const checkout = join(__dirname, 'checkout-server.sh'); - const calls = join(root, 'git-calls'); - writeFileSync(join(bin, 'python3'), `#!/bin/bash\ncat >/dev/null\nprintf '%s\\n' '${full}'\n`, { mode: 0o755 }); - writeFileSync(join(bin, 'git'), `#!/bin/bash\nprintf '%s\\n' "$*" >> "$GIT_CALLS"\nif [[ "$*" == *'rev-parse HEAD' ]]; then printf '%s\\n' '${full}'; fi\n`, { mode: 0o755 }); - const result = spawnSync('bash', [checkout, full.slice(0, 6), join(root, 'checkout')], { - env: { ...process.env, SERVER_REPOSITORY: 'apache/hugegraph', SERVER_FETCH_REF: full.slice(0, 6), - PATH: `${bin}:${process.env.PATH}`, GIT_CALLS: calls }, encoding: 'utf8' - }); - assert.equal(result.status, 0, result.stderr); - const transport = readFileSync(calls, 'utf8'); - assert.ok(transport.includes(`fetch --depth 1 -- origin ${full}\n`)); - assert.match(result.stdout, /Server source: apache\/hugegraph@aaaaaa/); -}); diff --git a/hugegraph-hubble/Dockerfile b/hugegraph-hubble/Dockerfile index a8e4d6988..b53408086 100644 --- a/hugegraph-hubble/Dockerfile +++ b/hugegraph-hubble/Dockerfile @@ -40,7 +40,7 @@ RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ SERVER_REPOSITORY=apache/hugegraph \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - d9abcd /opt/candidate-source /opt/candidate-m2; \ + d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 diff --git a/hugegraph-loader/Dockerfile b/hugegraph-loader/Dockerfile index bb541981b..99a71cc66 100644 --- a/hugegraph-loader/Dockerfile +++ b/hugegraph-loader/Dockerfile @@ -36,7 +36,7 @@ RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ SERVER_REPOSITORY=apache/hugegraph \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - d9abcd /opt/candidate-source /opt/candidate-m2; \ + d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 From 645d2548c082b97f4cd9b883429c05c8627e667b Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 22:27:41 +0800 Subject: [PATCH 28/38] fix(hubble): filter smoke archive extraction - retain archive member preflight validation - apply the native data filter during extraction - cover sequential links and valid internal links --- .../assembly/travis/run_live_hubble_smoke.py | 11 ++-- .../travis/test_run_live_hubble_smoke.py | 54 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py b/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py index 0f6031a96..ec264d5f0 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py +++ b/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py @@ -122,12 +122,17 @@ def assert_safe_tar_member(member, work_dir): raise RuntimeError(f"Unsafe tar link outside work dir: {member.name}") -def extract_tarball(tarball, work_dir): - with tarfile.open(tarball, "r:gz") as archive: +def extract_archive(tarball, work_dir): + with tarfile.open(tarball) as archive: members = archive.getmembers() for member in members: assert_safe_tar_member(member, work_dir) - archive.extractall(work_dir, members) + # Recheck each member against links already written during extraction. + archive.extractall(work_dir, members, filter="data") + + +def extract_tarball(tarball, work_dir): + extract_archive(tarball, work_dir) homes = [path for path in work_dir.iterdir() if path.is_dir() and path.name.startswith("apache-hugegraph-hubble-")] if not homes: diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py b/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py index 78eb8b957..752b682db 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py +++ b/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py @@ -17,6 +17,8 @@ # import importlib.util +import io +import tarfile import tempfile import unittest from pathlib import Path @@ -29,6 +31,58 @@ SPEC.loader.exec_module(SMOKE) +class ArchiveExtractionTest(unittest.TestCase): + + def test_sequential_symlink_escape_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + work = root / "runtime" + work.mkdir() + tarball = root / "escape.tar.gz" + with tarfile.open(tarball, "w:gz") as archive: + for name, target in (("pivot", "."), ("pivot/escape", "..")): + member = tarfile.TarInfo(name) + member.type = tarfile.SYMTYPE + member.linkname = target + archive.addfile(member) + member = tarfile.TarInfo("pivot/escape/escaped") + member.size = 7 + archive.addfile(member, io.BytesIO(b"outside")) + # Each entry passes the old preflight before any links exist. + with tarfile.open(tarball) as archive: + for member in archive.getmembers(): + SMOKE.assert_safe_tar_member(member, work) + + with self.assertRaises(tarfile.FilterError): + SMOKE.extract_archive(tarball, work) + self.assertFalse((root / "escaped").exists()) + + def test_internal_symbolic_and_hard_links_are_preserved(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + work = root / "runtime" + work.mkdir() + tarball = root / "hubble.tar.gz" + home_name = "apache-hugegraph-hubble-fixture" + with tarfile.open(tarball, "w:gz") as archive: + member = tarfile.TarInfo(f"{home_name}/lib/payload") + member.size = 7 + archive.addfile(member, io.BytesIO(b"fixture")) + for name, target, kind in ( + ("current", "lib", tarfile.SYMTYPE), + ("copy", f"{home_name}/lib/payload", tarfile.LNKTYPE)): + member = tarfile.TarInfo(f"{home_name}/{name}") + member.type = kind + member.linkname = target + archive.addfile(member) + + home = SMOKE.extract_tarball(tarball, work) + self.assertEqual(work / home_name, home) + self.assertTrue((home / "current").is_symlink()) + self.assertEqual(b"fixture", (home / "current/payload").read_bytes()) + self.assertEqual(b"fixture", (home / "copy").read_bytes()) + + class HubbleReadinessTest(unittest.TestCase): def test_configure_endpoint_uses_registered_server_options(self): From bb67264e29264007fbe63d846380c079345b92bb Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 22:41:47 +0800 Subject: [PATCH 29/38] fix(ci): advance the UTF-8 SDK baseline - use the merged ASF request encoding fix - align workflow and image source identities - retain full machine locks and short documentation --- .github/actions/setup-candidate-sdk/action.yml | 4 ++-- .github/scripts/verify_candidate_image_sdk.py | 2 +- .github/workflows/ci.yml | 4 ++-- docs/ci.md | 2 +- docs/java17-migration.md | 2 +- hugegraph-hubble/Dockerfile | 2 +- hugegraph-loader/Dockerfile | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index 7bef989fb..67b5bddf1 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -7,10 +7,10 @@ inputs: default: 'apache/hugegraph' ref: description: 'Immutable candidate source commit' - default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' + default: 'e62c961e00221569d4f955abbadf60faee45b283' expected-commit: description: 'Locked candidate source commit' - default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' + default: 'e62c961e00221569d4f955abbadf60faee45b283' outputs: repository: diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index 1ee62069e..291ae46d6 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -25,7 +25,7 @@ from pathlib import Path REPOSITORY = "apache/hugegraph" -COMMIT = "d9abcd4317fb36128e7e4d209139ec7f3a28cdfc" +COMMIT = "e62c961e00221569d4f955abbadf60faee45b283" REQUIRED_MODULES = { "pom.xml", "hugegraph-commons/pom.xml", "hugegraph-server/pom.xml", "hugegraph-pd/pom.xml", "hugegraph-store/pom.xml", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c4b38d6a..1813c0214 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,8 +86,8 @@ jobs: uses: ./.github/actions/get-hugegraph-commit with: repository: apache/hugegraph - ref: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc - expected-commit: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc + ref: e62c961e00221569d4f955abbadf60faee45b283 + expected-commit: e62c961e00221569d4f955abbadf60faee45b283 - name: Record external test inputs env: NEEDS_SERVER: ${{ steps.scope.outputs.needsFixture }} diff --git a/docs/ci.md b/docs/ci.md index 3479638b7..9c1770af8 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -38,7 +38,7 @@ Each image job builds the checkout's actual Dockerfile, records its image ID and Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement affected-module selection without adding unrelated modules. Their results remain advisory. -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. CI locks the complete immutable source commit (shown as `d9abcd` here), so later master updates do not change the build inputs. The SDK action, CI fixtures, image defaults and packaging verifier use the same locked source commit. +The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. CI locks the complete immutable source commit (shown as `e62c96` here), so later master updates do not change the build inputs. The SDK action, CI fixtures, image defaults and packaging verifier use the same locked source commit. The released Server package and Hubble's candidate fixture are each built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly POM versions before installation, including the root parent, so dependency and license metadata readers consume concrete coordinates without changing the locked source POMs. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. diff --git a/docs/java17-migration.md b/docs/java17-migration.md index b07f3b36f..cb0353625 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -6,7 +6,7 @@ Toolchain builds, tests and applications use Java 17. The released Server 1.7 co ## Build the locked candidate -The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses the verified `apache/hugegraph` master baseline (shown as `d9abcd` here), storing and fetching its complete immutable commit identity. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. +The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses the verified `apache/hugegraph` master baseline (shown as `e62c96` here), storing and fetching its complete immutable commit identity. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. Use an explicit JDK directory and check both Java and Maven. On systems where Java 17 is not registered, a system JDK selector can return another installed version. Checking the generated class version alone does not identify the JVM that built or tested it. diff --git a/hugegraph-hubble/Dockerfile b/hugegraph-hubble/Dockerfile index b53408086..a57653ef0 100644 --- a/hugegraph-hubble/Dockerfile +++ b/hugegraph-hubble/Dockerfile @@ -40,7 +40,7 @@ RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ SERVER_REPOSITORY=apache/hugegraph \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ + e62c961e00221569d4f955abbadf60faee45b283 /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 diff --git a/hugegraph-loader/Dockerfile b/hugegraph-loader/Dockerfile index 99a71cc66..1985afe64 100644 --- a/hugegraph-loader/Dockerfile +++ b/hugegraph-loader/Dockerfile @@ -36,7 +36,7 @@ RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ SERVER_REPOSITORY=apache/hugegraph \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ + e62c961e00221569d4f955abbadf60faee45b283 /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 From cecdb33b55d532f3e688c0707fd6acfe440201ba Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 7 Oct 2026 23:56:36 +0800 Subject: [PATCH 30/38] fix(ci): use the official server 1.7 release - pin the ASF binary release URL and official SHA-512 - verify release identity and both archive checksums on cache reuse - preserve Java 17 candidate builds and existing service startup - update offline fixture contracts for release downloads and failures --- .../actions/setup-hugegraph-server/action.yml | 16 ++- .../actions/setup-hugegraph-server/fixture.sh | 29 ++--- .../setup-hugegraph-server/manifest.py | 13 +++ .../actions/setup-hugegraph-server/release.py | 80 +++++++++++++ .../setup-hugegraph-server/runtime_test.py | 109 +++++++++++++++--- 5 files changed, 216 insertions(+), 31 deletions(-) create mode 100644 .github/actions/setup-hugegraph-server/release.py diff --git a/.github/actions/setup-hugegraph-server/action.yml b/.github/actions/setup-hugegraph-server/action.yml index 2077e6519..8556ce21e 100644 --- a/.github/actions/setup-hugegraph-server/action.yml +++ b/.github/actions/setup-hugegraph-server/action.yml @@ -1,5 +1,5 @@ name: Setup HugeGraph Server -description: Build an immutable fixture once, or start an independent test service +description: Prepare an immutable release or candidate fixture, or start an independent test service inputs: mode: description: 'standalone (legacy), build (upload), or start (download)' @@ -19,6 +19,9 @@ inputs: java-version: description: JDK major used for the build and runtime default: '11' + release-version: + description: Fixed official Server release used by Java 11 compatibility fixtures + default: '1.7.0' java-home: description: Optional fixture JDK scoped to this subprocess; keeps Toolchain JDK unchanged default: '' @@ -37,12 +40,12 @@ runs: with: name: ${{ inputs.artifact-name }} path: ${{ runner.temp }}/hugegraph-fixture - - name: Restore exact Server build cache + - name: Restore exact Server fixture cache if: inputs.mode == 'build' uses: actions/cache@v4 with: path: ~/hugegraph-fixture-cache - key: server-fixture-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.repository }}-${{ inputs.commit-id }}-jdk${{ inputs.java-version }}-${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} + key: server-fixture-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.repository }}-${{ inputs.commit-id }}-release${{ inputs.release-version }}-jdk${{ inputs.java-version }}-${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-hugegraph-server/release.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} - name: Prepare fixture or start service shell: bash env: @@ -50,9 +53,10 @@ runs: FIXTURE_REPOSITORY: ${{ inputs.repository }} FIXTURE_COMMIT: ${{ inputs.commit-id }} FIXTURE_REF: ${{ inputs.fetch-ref }} - FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} + FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-hugegraph-server/release.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} FIXTURE_PLATFORM: ${{ runner.os }}-${{ runner.arch }} FIXTURE_JAVA: ${{ inputs.java-version }} + FIXTURE_RELEASE_VERSION: ${{ inputs.release-version }} FIXTURE_JAVA_HOME: ${{ inputs.java-home }} ISOLATE_FIXTURE_REPO: ${{ inputs.isolate-maven-repo }} FIXTURE_DIR: ${{ runner.temp }}/hugegraph-fixture @@ -85,6 +89,10 @@ runs: export TRAVIS_DIR server_build=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-build.XXXXXX") cd "$server_build" + if [[ "$FIXTURE_MODE" == standalone ]]; then + FIXTURE_MODE=build bash "$FIXTURE_ACTION/fixture.sh" + export FIXTURE_MODE=start + fi if [[ -f "$TRAVIS_DIR/install-hugegraph-from-source.sh" ]]; then bash "$TRAVIS_DIR/install-hugegraph-from-source.sh" "$FIXTURE_COMMIT" else diff --git a/.github/actions/setup-hugegraph-server/fixture.sh b/.github/actions/setup-hugegraph-server/fixture.sh index 60df1d70b..b3c23196a 100644 --- a/.github/actions/setup-hugegraph-server/fixture.sh +++ b/.github/actions/setup-hugegraph-server/fixture.sh @@ -22,6 +22,7 @@ FIXTURE_REPOSITORY=${FIXTURE_REPOSITORY:-${SERVER_REPOSITORY:-apache/hugegraph}} FIXTURE_COMMIT=${FIXTURE_COMMIT:-${1:-}} FIXTURE_REF=${FIXTURE_REF:-$FIXTURE_COMMIT} FIXTURE_JAVA=${FIXTURE_JAVA:-11} +FIXTURE_RELEASE_VERSION=${FIXTURE_RELEASE_VERSION:-1.7.0} FIXTURE_DIR=${FIXTURE_DIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-fixture} FIXTURE_HELPER=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/manifest.py FIXTURE_BUILD_INPUTS=${FIXTURE_BUILD_INPUTS:-$(python3 "$FIXTURE_HELPER" inputs)} @@ -29,13 +30,16 @@ FIXTURE_PLATFORM=${FIXTURE_PLATFORM:-$(uname -sm)} if [[ "$FIXTURE_JAVA" == 17 ]]; then FIXTURE_BUILD_COMMAND=candidate-sdk-install else - FIXTURE_BUILD_COMMAND='package -DskipTests -Dmaven.javadoc.skip=true -ntp' + FIXTURE_BUILD_COMMAND="asf-release-$FIXTURE_RELEASE_VERSION" fi -FIXTURE_CONFIG="$FIXTURE_BUILD_COMMAND;schema=1;inputs=$FIXTURE_BUILD_INPUTS;platform=$FIXTURE_PLATFORM" -export FIXTURE_REPOSITORY FIXTURE_COMMIT FIXTURE_JAVA FIXTURE_CONFIG FIXTURE_DIR +FIXTURE_CONFIG="$FIXTURE_BUILD_COMMAND;schema=2;inputs=$FIXTURE_BUILD_INPUTS;platform=$FIXTURE_PLATFORM" +export FIXTURE_REPOSITORY FIXTURE_COMMIT FIXTURE_JAVA FIXTURE_CONFIG FIXTURE_DIR FIXTURE_RELEASE_VERSION [[ "$FIXTURE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] [[ "$FIXTURE_COMMIT" =~ ^[0-9a-f]{40}$ ]] -[[ "$FIXTURE_JAVA" =~ ^[0-9]+$ ]] +[[ "$FIXTURE_JAVA" == 11 || "$FIXTURE_JAVA" == 17 ]] || { + echo 'Fixture requires JDK 11 for the official release or JDK 17 for the candidate SDK' >&2 + exit 1 +} fixture_validate_java() { actual_java=$(java -version 2>&1 | sed -n 's/.*version "\([0-9][0-9]*\).*".*/\1/p' | head -1) @@ -55,27 +59,26 @@ fixture_build() { fi rm -f "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" fixture_validate_java - source_dir="$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-source.XXXXXX")/source" if [[ "$FIXTURE_JAVA" == 17 ]]; then + source_dir="$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-source.XXXXXX")/source" # Reuse the verified SDK build's same-source distribution and isolated repository. candidate_repo=$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-sdk-m2.XXXXXX") SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \ bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ "$FIXTURE_COMMIT" "$source_dir" "$candidate_repo" + archive=$(find "$source_dir/hugegraph-server" -maxdepth 1 -name 'apache-hugegraph-*.tar.gz' -print) + [[ -n "$archive" && -f "$archive" ]] + cp "$archive" "$FIXTURE_DIR/server.tar.gz" + export FIXTURE_ARCHIVE_NAME=${archive##*/} else - SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \ - bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/checkout-server.sh" "$FIXTURE_COMMIT" "$source_dir" - (cd "$source_dir" && mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp) + python3 "${FIXTURE_HELPER%/*}/release.py" download + export FIXTURE_ARCHIVE_NAME=$(python3 "${FIXTURE_HELPER%/*}/release.py" name) fi - archive=$(find "$source_dir/hugegraph-server" -maxdepth 1 -name 'apache-hugegraph-*.tar.gz' -print) - [[ -n "$archive" && -f "$archive" ]] - cp "$archive" "$FIXTURE_DIR/server.tar.gz" - export FIXTURE_ARCHIVE_NAME=${archive##*/} python3 "$FIXTURE_HELPER" create fixture_verify mkdir -p "$cache_dir" cp "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" "$cache_dir/" - rm -rf "$source_dir" + if [[ -n "${source_dir:-}" ]]; then rm -rf "$source_dir"; fi if [[ -n "$candidate_repo" ]]; then rm -rf "$candidate_repo"; fi } fixture_copy_archive() { diff --git a/.github/actions/setup-hugegraph-server/manifest.py b/.github/actions/setup-hugegraph-server/manifest.py index c7eab2468..2ca5747a6 100644 --- a/.github/actions/setup-hugegraph-server/manifest.py +++ b/.github/actions/setup-hugegraph-server/manifest.py @@ -23,12 +23,15 @@ import re import sys +import release + if sys.argv[1] == "inputs": repo = pathlib.Path(__file__).resolve().parents[3] patterns = ( '.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', + '.github/actions/setup-hugegraph-server/release.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', @@ -46,6 +49,12 @@ root = pathlib.Path(os.environ["FIXTURE_DIR"]) identity = {key.lower(): os.environ["FIXTURE_" + key] for key in ("REPOSITORY", "COMMIT", "JAVA", "CONFIG")} +if identity["java"] == "11": + identity.update(release.identity(os.environ.get("FIXTURE_RELEASE_VERSION", release.VERSION))) +else: + identity.update(source_kind="candidate-sdk") + + def digest(): h = hashlib.sha256() with (root / "server.tar.gz").open("rb") as stream: @@ -53,6 +62,8 @@ def digest(): h.update(chunk) return h.hexdigest() try: + if identity["java"] == "11": + release.verify(root / "server.tar.gz") if sys.argv[1] == "create": data = dict(identity, archive_name=os.environ["FIXTURE_ARCHIVE_NAME"], sha256=digest()) (root / "manifest.json").write_text(json.dumps(data, sort_keys=True) + "\n") @@ -64,6 +75,8 @@ def digest(): raise ValueError("fixture origin, commit, JDK or build configuration mismatch") if not re.fullmatch(r"apache-hugegraph-[A-Za-z0-9_.-]+\.tar\.gz", data.get("archive_name", "")): raise ValueError("invalid archive name") + if identity["java"] == "11" and data.get("archive_name") != release.ARCHIVE_NAME: + raise ValueError("official Server release archive name mismatch") if data.get("sha256") != digest(): raise ValueError("fixture checksum mismatch") if sys.argv[1] == "name": diff --git a/.github/actions/setup-hugegraph-server/release.py b/.github/actions/setup-hugegraph-server/release.py new file mode 100644 index 000000000..2d348b995 --- /dev/null +++ b/.github/actions/setup-hugegraph-server/release.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +"""Pinned ASF binary release identity and bounded download, without source fallback.""" +import hashlib +import os +import pathlib +import subprocess +import sys +import tempfile + +VERSION = "1.7.0" +ARCHIVE_NAME = "apache-hugegraph-incubating-1.7.0.tar.gz" +SOURCE_URL = "https://archive.apache.org/dist/incubator/hugegraph/1.7.0/" + ARCHIVE_NAME +# Verified against SOURCE_URL + ".sha512" in the official ASF archive. +SHA512 = ("ba093203e817f17582895ff10ceb0458498c886ec7a687fd0f3e5f56ba739454" + "ec928a592bdad8a3685651dc9f8ef904d082f9530d4aae6a83b5d9d9626ad13f") + + +def identity(version): + if version != VERSION: + raise ValueError("unsupported Server release version: " + version) + return dict(source_kind="asf-release", release_version=VERSION, + source_url=SOURCE_URL, official_sha512=SHA512) + + +def digest(path): + result = hashlib.sha512() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + result.update(chunk) + return result.hexdigest() + + +def verify(path): + if not path.is_file() or path.is_symlink() or digest(path) != SHA512: + raise ValueError("official Server release SHA-512 mismatch") + + +def download(directory): + directory.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=".server-release-", dir=directory) + os.close(descriptor) + path = pathlib.Path(temporary) + try: + subprocess.run(["curl", "--fail", "--location", "--show-error", "--silent", + "--connect-timeout", "20", "--max-time", "900", "--retry", "2", + "--proto", "=https", "--proto-redir", "=https", + "--output", str(path), SOURCE_URL], check=True) + verify(path) + path.replace(directory / "server.tar.gz") + finally: + path.unlink(missing_ok=True) + + +if __name__ == "__main__": + try: + identity(os.environ.get("FIXTURE_RELEASE_VERSION", VERSION)) + if sys.argv[1] == "download": + download(pathlib.Path(os.environ["FIXTURE_DIR"])) + elif sys.argv[1] == "name": + print(ARCHIVE_NAME) + else: + raise ValueError("invalid release helper command") + except (OSError, ValueError, subprocess.CalledProcessError) as error: + sys.exit(str(error)) diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index 8ef582267..3830fe5fd 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -18,10 +18,12 @@ """Exercise fixture identity, bounded downloads and service readiness offline.""" import hashlib import json +import importlib.util import os from pathlib import Path import subprocess import tempfile +import shutil import unittest HERE = Path(__file__).resolve().parent @@ -33,6 +35,19 @@ def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.dir = Path(self.temp.name) + # Offline archives use a test-local checksum pin, never a production bypass. + self.action = self.dir / 'action' + self.action.mkdir() + for name in ('fixture.sh', 'manifest.py', 'release.py', 'service-wait.sh'): + shutil.copyfile(HERE / name, self.action / name) + spec = importlib.util.spec_from_file_location('fixture_release', HERE / 'release.py') + self.release = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.release) + checksum = hashlib.sha512(b'archive').hexdigest() + release_code = (self.action / 'release.py').read_text() + release_code = release_code.replace(self.release.SHA512[:64], checksum[:64]) + release_code = release_code.replace(self.release.SHA512[64:], checksum[64:]) + (self.action / 'release.py').write_text(release_code) self.bin = self.dir / 'bin' self.bin.mkdir() (self.bin/'java').write_text("#!/bin/bash\necho 'openjdk version \"11.0.1\"' >&2\n") @@ -41,7 +56,8 @@ def setUp(self): FIXTURE_DIR=str(self.dir / 'fixture'), FIXTURE_REPOSITORY='apache/hugegraph', FIXTURE_COMMIT=SHA, FIXTURE_JAVA='11', FIXTURE_BUILD_INPUTS='test-build-contract', FIXTURE_PLATFORM='Linux-X64', - FIXTURE_CONFIG='package -DskipTests -Dmaven.javadoc.skip=true -ntp;schema=1;inputs=test-build-contract;platform=Linux-X64') + FIXTURE_RELEASE_VERSION='1.7.0', + FIXTURE_CONFIG='asf-release-1.7.0;schema=2;inputs=test-build-contract;platform=Linux-X64') Path(self.env['FIXTURE_DIR']).mkdir() def command(self, name, body): script = self.bin / name @@ -54,27 +70,36 @@ def manifest(self, **extra): root = Path(self.env['FIXTURE_DIR']) data = {k.lower(): self.env['FIXTURE_'+k] for k in ('REPOSITORY','COMMIT','JAVA','CONFIG')} (root/'server.tar.gz').write_bytes(b'archive') - data.update(archive_name='apache-hugegraph-fixture.tar.gz', sha256=hashlib.sha256(b'archive').hexdigest()) + if extra.get('java', self.env['FIXTURE_JAVA']) == '11': + data.update(self.release.identity('1.7.0')) + data['official_sha512'] = hashlib.sha512(b'archive').hexdigest() + name = self.release.ARCHIVE_NAME + else: + data.update(source_kind='candidate-sdk') + name = 'apache-hugegraph-fixture.tar.gz' + data.update(archive_name=name, sha256=hashlib.sha256(b'archive').hexdigest()) data.update(extra) (root/'manifest.json').write_text(json.dumps(data)) def test_identity_checksum_and_regular_files(self): self.manifest() env = dict(self.env, FIXTURE_MODE='start') - self.assertEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) - for field, wrong in [('repository','fork/hugegraph'), ('commit','b'*40), ('java','17'), ('config','different'), ('sha256','bad')]: + self.assertEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) + for field, wrong in [('repository','fork/hugegraph'), ('commit','b'*40), ('java','17'), + ('config','different'), ('source_url','https://example.invalid/release'), + ('release_version','1.8.0'), ('official_sha512','bad'), ('sha256','bad')]: self.manifest(**{field:wrong}) - self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0, field) + self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0, field) self.manifest() self.command('java', "echo 'openjdk version \"17.0.1\"' >&2\n") - self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) self.command('java', "echo 'openjdk version \"11.0.1\"' >&2\n") self.manifest(archive_name='../escape.tar.gz') - self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) self.manifest() archive = Path(self.env['FIXTURE_DIR'])/'server.tar.gz' archive.unlink() archive.symlink_to(self.dir/'missing') - self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) def test_composite_step_isolates_fixture_jdk_from_caller(self): action = (HERE / 'action.yml').read_text() block = action.split(' run: |\n', 1)[1].split(' - name:', 1)[0] @@ -103,7 +128,7 @@ def test_composite_step_isolates_fixture_jdk_from_caller(self): env = dict(self.env, JAVA_HOME=str(caller_jdk), PATH=f'{caller_jdk}/bin:{self.env["PATH"]}', FIXTURE_JAVA=str(fixture_version), FIXTURE_JAVA_HOME=str(fixture_jdk), - FIXTURE_MODE='start', FIXTURE_ARTIFACT='shared', FIXTURE_ACTION=str(HERE), + FIXTURE_MODE='start', FIXTURE_ARTIFACT='shared', FIXTURE_ACTION=str(self.action), ISOLATE_FIXTURE_REPO='false', TRAVIS_DIR=str(travis), RUNNER_TEMP=str(self.dir)) result = subprocess.run( ['bash', '-c', 'bash "$HOME/action-step.sh" && java -version 2> "$HOME/caller-java"'], @@ -200,15 +225,71 @@ def test_exact_cache_hit_never_builds_or_starts(self): (Path(self.env['FIXTURE_DIR'])/name).unlink() self.command('git', 'echo forbidden-git >> "$HOME/forbidden"; exit 99\n') self.command('mvn', 'echo forbidden-build >> "$HOME/forbidden"; exit 99\n') - result = self.run_script(HERE/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.command('curl', 'echo download >> "$HOME/downloads"; exit 22\n') + result = self.run_script(self.action/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) self.assertEqual(result.returncode, 0, result.stderr) self.assertFalse((self.dir/'forbidden').exists()) - self.assertFalse((self.dir/'apache-hugegraph-fixture.tar.gz').exists()) - # An incomplete exact cache must rebuild, rather than poison future runs. + self.assertFalse((self.dir/self.release.ARCHIVE_NAME).exists()) + # An incomplete release cache must download again, never rebuild old source. (cache/'manifest.json').unlink() - result = self.run_script(HERE/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + result = self.run_script(self.action/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) self.assertNotEqual(result.returncode, 0) - self.assertTrue((self.dir/'forbidden').exists()) + self.assertFalse((self.dir/'forbidden').exists()) + self.assertTrue((self.dir/'downloads').exists()) + + def test_release_pin_matches_official_identity_and_rejects_wrong_bytes(self): + self.assertEqual('1.7.0', self.release.VERSION) + self.assertEqual('https://archive.apache.org/dist/incubator/hugegraph/1.7.0/' + + self.release.ARCHIVE_NAME, self.release.SOURCE_URL) + self.assertEqual(128, len(self.release.SHA512)) + self.assertEqual(self.release.SHA512, + self.release.identity('1.7.0')['official_sha512']) + with self.assertRaises(ValueError): + self.release.identity('1.8.0') + archive = self.dir / 'invalid-release.tar.gz' + archive.write_bytes(b'archive') + with self.assertRaisesRegex(ValueError, 'SHA-512 mismatch'): + self.release.verify(archive) + + def test_release_download_keeps_official_identity_and_never_builds(self): + self.command('curl', 'printf "%s\\n" "$@" > "$HOME/download-args"\n' + 'while [[ $# -gt 0 ]]; do\n' + ' if [[ "$1" == --output ]]; then printf archive > "$2"; exit 0; fi\n' + ' shift\ndone\nexit 1\n') + for name in ('git', 'mvn'): + self.command(name, 'echo forbidden >> "$HOME/forbidden"; exit 99\n') + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertEqual(0, result.returncode, result.stderr) + self.assertFalse((self.dir / 'forbidden').exists()) + manifest = json.loads((Path(self.env['FIXTURE_DIR']) / 'manifest.json').read_text()) + self.assertEqual('asf-release', manifest['source_kind']) + self.assertEqual(self.release.SOURCE_URL, manifest['source_url']) + self.assertEqual(hashlib.sha512(b'archive').hexdigest(), manifest['official_sha512']) + args = (self.dir / 'download-args').read_text() + self.assertIn('--max-time\n900', args) + self.assertIn('--proto\n=https', args) + self.assertIn(self.release.SOURCE_URL, args) + + def test_release_failure_never_falls_back_to_source_or_publishes_cache(self): + self.command('curl', 'echo download >> "$HOME/downloads"; exit 22\n') + for name in ('git', 'mvn'): + self.command(name, 'echo forbidden >> "$HOME/forbidden"; exit 99\n') + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertNotEqual(0, result.returncode) + self.assertFalse((self.dir / 'forbidden').exists()) + self.assertFalse((self.dir / 'hugegraph-fixture-cache').exists()) + self.assertEqual([], list(Path(self.env['FIXTURE_DIR']).iterdir())) + + def test_release_rejects_tampering_even_with_updated_local_sha256(self): + self.manifest() + fixture = Path(self.env['FIXTURE_DIR']) + (fixture / 'server.tar.gz').write_bytes(b'changed archive') + manifest = json.loads((fixture / 'manifest.json').read_text()) + manifest['sha256'] = hashlib.sha256(b'changed archive').hexdigest() + (fixture / 'manifest.json').write_text(json.dumps(manifest)) + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='start')) + self.assertNotEqual(0, result.returncode) + self.assertIn('official Server release SHA-512 mismatch', result.stderr) if __name__ == '__main__': From ed75a791b35dc56adfeda62ba2ad250db09be35f Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 00:08:01 +0800 Subject: [PATCH 31/38] fix(ci): bind release fixtures to source identity - fix the official 1.7.0 source repository and commit - reject mismatched callers before cache reads or downloads - validate and record release source identity in manifests - cover false source claims in offline fixture contracts --- .../actions/setup-hugegraph-server/action.yml | 4 +-- .../actions/setup-hugegraph-server/fixture.sh | 5 ++++ .../setup-hugegraph-server/manifest.py | 12 +++++--- .../actions/setup-hugegraph-server/release.py | 16 +++++++--- .../setup-hugegraph-server/runtime_test.py | 29 +++++++++++++++++-- 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/.github/actions/setup-hugegraph-server/action.yml b/.github/actions/setup-hugegraph-server/action.yml index 8556ce21e..979a5f19d 100644 --- a/.github/actions/setup-hugegraph-server/action.yml +++ b/.github/actions/setup-hugegraph-server/action.yml @@ -8,13 +8,13 @@ inputs: description: Workflow-local immutable fixture artifact name default: '' repository: - description: GitHub owner/repository containing the Server source + description: Server source repository (Java 11 release requires apache/hugegraph) default: apache/hugegraph fetch-ref: description: Fetchable source ref (must resolve to commit-id) default: '' commit-id: - description: Exact Server commit SHA + description: Exact Server commit SHA (Java 11 release requires the locked 1.7.0 commit) required: true java-version: description: JDK major used for the build and runtime diff --git a/.github/actions/setup-hugegraph-server/fixture.sh b/.github/actions/setup-hugegraph-server/fixture.sh index b3c23196a..375045811 100644 --- a/.github/actions/setup-hugegraph-server/fixture.sh +++ b/.github/actions/setup-hugegraph-server/fixture.sh @@ -41,6 +41,11 @@ export FIXTURE_REPOSITORY FIXTURE_COMMIT FIXTURE_JAVA FIXTURE_CONFIG FIXTURE_DIR exit 1 } +if [[ "$FIXTURE_JAVA" == 11 ]]; then + # Reject caller/source mismatch before restoring a cache or downloading bytes. + python3 "${FIXTURE_HELPER%/*}/release.py" validate +fi + fixture_validate_java() { actual_java=$(java -version 2>&1 | sed -n 's/.*version "\([0-9][0-9]*\).*".*/\1/p' | head -1) [[ "$actual_java" == "$FIXTURE_JAVA" ]] || { echo 'Fixture JDK mismatch' >&2; return 1; } diff --git a/.github/actions/setup-hugegraph-server/manifest.py b/.github/actions/setup-hugegraph-server/manifest.py index 2ca5747a6..0b13a7514 100644 --- a/.github/actions/setup-hugegraph-server/manifest.py +++ b/.github/actions/setup-hugegraph-server/manifest.py @@ -49,10 +49,14 @@ root = pathlib.Path(os.environ["FIXTURE_DIR"]) identity = {key.lower(): os.environ["FIXTURE_" + key] for key in ("REPOSITORY", "COMMIT", "JAVA", "CONFIG")} -if identity["java"] == "11": - identity.update(release.identity(os.environ.get("FIXTURE_RELEASE_VERSION", release.VERSION))) -else: - identity.update(source_kind="candidate-sdk") +try: + if identity["java"] == "11": + identity.update(release.identity(os.environ.get("FIXTURE_RELEASE_VERSION", release.VERSION), + identity["repository"], identity["commit"])) + else: + identity.update(source_kind="candidate-sdk") +except ValueError as error: + sys.exit(str(error)) def digest(): diff --git a/.github/actions/setup-hugegraph-server/release.py b/.github/actions/setup-hugegraph-server/release.py index 2d348b995..0e1b7472b 100644 --- a/.github/actions/setup-hugegraph-server/release.py +++ b/.github/actions/setup-hugegraph-server/release.py @@ -24,6 +24,8 @@ import tempfile VERSION = "1.7.0" +REPOSITORY = "apache/hugegraph" +COMMIT = "b12425c2032bf0d21a97b8221f42a18055c2982f" ARCHIVE_NAME = "apache-hugegraph-incubating-1.7.0.tar.gz" SOURCE_URL = "https://archive.apache.org/dist/incubator/hugegraph/1.7.0/" + ARCHIVE_NAME # Verified against SOURCE_URL + ".sha512" in the official ASF archive. @@ -31,10 +33,13 @@ "ec928a592bdad8a3685651dc9f8ef904d082f9530d4aae6a83b5d9d9626ad13f") -def identity(version): +def identity(version, repository=REPOSITORY, commit=COMMIT): if version != VERSION: raise ValueError("unsupported Server release version: " + version) + if repository != REPOSITORY or commit != COMMIT: + raise ValueError("official Server release requires " + REPOSITORY + "@" + COMMIT) return dict(source_kind="asf-release", release_version=VERSION, + source_repository=REPOSITORY, source_commit=COMMIT, source_url=SOURCE_URL, official_sha512=SHA512) @@ -69,12 +74,15 @@ def download(directory): if __name__ == "__main__": try: - identity(os.environ.get("FIXTURE_RELEASE_VERSION", VERSION)) - if sys.argv[1] == "download": + identity(os.environ.get("FIXTURE_RELEASE_VERSION", VERSION), + os.environ["FIXTURE_REPOSITORY"], os.environ["FIXTURE_COMMIT"]) + if sys.argv[1] == "validate": + pass + elif sys.argv[1] == "download": download(pathlib.Path(os.environ["FIXTURE_DIR"])) elif sys.argv[1] == "name": print(ARCHIVE_NAME) else: raise ValueError("invalid release helper command") - except (OSError, ValueError, subprocess.CalledProcessError) as error: + except (OSError, ValueError, KeyError, subprocess.CalledProcessError) as error: sys.exit(str(error)) diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index 3830fe5fd..2814baab4 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -28,7 +28,7 @@ HERE = Path(__file__).resolve().parent ROOT = HERE.parents[2] -SHA = 'a' * 40 +SHA = 'b12425c2032bf0d21a97b8221f42a18055c2982f' class RuntimeTests(unittest.TestCase): def setUp(self): @@ -86,7 +86,8 @@ def test_identity_checksum_and_regular_files(self): self.assertEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) for field, wrong in [('repository','fork/hugegraph'), ('commit','b'*40), ('java','17'), ('config','different'), ('source_url','https://example.invalid/release'), - ('release_version','1.8.0'), ('official_sha512','bad'), ('sha256','bad')]: + ('release_version','1.8.0'), ('source_repository','fork/hugegraph'), + ('source_commit','b'*40), ('official_sha512','bad'), ('sha256','bad')]: self.manifest(**{field:wrong}) self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0, field) self.manifest() @@ -241,6 +242,8 @@ def test_release_pin_matches_official_identity_and_rejects_wrong_bytes(self): self.assertEqual('1.7.0', self.release.VERSION) self.assertEqual('https://archive.apache.org/dist/incubator/hugegraph/1.7.0/' + self.release.ARCHIVE_NAME, self.release.SOURCE_URL) + self.assertEqual('apache/hugegraph', self.release.REPOSITORY) + self.assertEqual(SHA, self.release.COMMIT) self.assertEqual(128, len(self.release.SHA512)) self.assertEqual(self.release.SHA512, self.release.identity('1.7.0')['official_sha512']) @@ -251,6 +254,26 @@ def test_release_pin_matches_official_identity_and_rejects_wrong_bytes(self): with self.assertRaisesRegex(ValueError, 'SHA-512 mismatch'): self.release.verify(archive) + def test_release_rejects_wrong_caller_source_before_cache_or_download(self): + for name in ('curl', 'git', 'mvn'): + self.command(name, 'echo forbidden >> "$HOME/forbidden"; exit 99\n') + for field, wrong in (('FIXTURE_REPOSITORY', 'fork/hugegraph'), + ('FIXTURE_COMMIT', 'a' * 40)): + with self.subTest(field=field): + env = dict(self.env, FIXTURE_MODE='build', **{field: wrong}) + result = self.run_script(self.action / 'fixture.sh', env=env) + self.assertNotEqual(0, result.returncode) + self.assertIn('official Server release requires', result.stderr) + self.assertFalse((self.dir / 'forbidden').exists()) + self.assertFalse((self.dir / 'hugegraph-fixture-cache').exists()) + self.assertFalse((Path(self.env['FIXTURE_DIR']) / 'manifest.json').exists()) + # Direct manifest callers must reject the same false source claim. + result = subprocess.run(['python3', str(self.action / 'manifest.py'), 'create'], + env=env, text=True, capture_output=True, timeout=20) + self.assertNotEqual(0, result.returncode) + self.assertIn('official Server release requires', result.stderr) + self.assertFalse((Path(self.env['FIXTURE_DIR']) / 'manifest.json').exists()) + def test_release_download_keeps_official_identity_and_never_builds(self): self.command('curl', 'printf "%s\\n" "$@" > "$HOME/download-args"\n' 'while [[ $# -gt 0 ]]; do\n' @@ -263,6 +286,8 @@ def test_release_download_keeps_official_identity_and_never_builds(self): self.assertFalse((self.dir / 'forbidden').exists()) manifest = json.loads((Path(self.env['FIXTURE_DIR']) / 'manifest.json').read_text()) self.assertEqual('asf-release', manifest['source_kind']) + self.assertEqual(self.release.REPOSITORY, manifest['source_repository']) + self.assertEqual(self.release.COMMIT, manifest['source_commit']) self.assertEqual(self.release.SOURCE_URL, manifest['source_url']) self.assertEqual(hashlib.sha512(b'archive').hexdigest(), manifest['official_sha512']) args = (self.dir / 'download-args').read_text() From 0c70206809310da98e60e67aecd9ee641574a716 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 00:12:02 +0800 Subject: [PATCH 32/38] fix(ci): isolate cached SDK inputs - reuse third-party Maven inputs without HugeGraph SDKs - remove the duplicate Hubble Maven cache - correct candidate build and Docker context instructions --- .../actions/setup-candidate-sdk/action.yml | 8 +++++++ .github/actions/setup-java-env/action.yml | 4 ++++ .github/workflows/hubble-ci.yml | 9 -------- AGENTS.md | 6 +++-- README.md | 2 +- docs/ci.md | 6 ++--- hugegraph-hubble/README.md | 17 +++++++++----- hugegraph-loader/README.md | 22 +++++++++---------- 8 files changed, 41 insertions(+), 33 deletions(-) diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index 67b5bddf1..02a52e556 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -56,8 +56,16 @@ runs: candidate_repo=$(mktemp -d "$RUNNER_TEMP/hugegraph-candidate-m2.XXXXXX") server_build=$(mktemp -d "$RUNNER_TEMP/hugegraph-candidate-build.XXXXXX") source_dir="$server_build/source" + if [[ -d "$HOME/.m2/repository" ]]; then + rsync -a --exclude='/org/apache/hugegraph/' --exclude='/candidate-sdk-manifest.json' \ + "$HOME/.m2/repository/" "$candidate_repo/" + fi bash "$GITHUB_WORKSPACE/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ "$SERVER_COMMIT" "$source_dir" "$candidate_repo" + # Retain downloaded third-party inputs for the existing Maven cache. + mkdir -p "$HOME/.m2/repository" + rsync -a --exclude='/org/apache/hugegraph/' --exclude='/candidate-sdk-manifest.json' \ + "$candidate_repo/" "$HOME/.m2/repository/" archives=("$source_dir"/hugegraph-server/apache-hugegraph-*.tar.gz) if [[ ${#archives[@]} -ne 1 || ! -f "${archives[0]}" ]]; then printf 'Expected one candidate server archive, found: %s\n' "${archives[*]}" >&2 diff --git a/.github/actions/setup-java-env/action.yml b/.github/actions/setup-java-env/action.yml index 517546cd8..a1e522ab7 100644 --- a/.github/actions/setup-java-env/action.yml +++ b/.github/actions/setup-java-env/action.yml @@ -23,12 +23,16 @@ runs: cache-dependency-path: | **/pom.xml .github/configs/settings.xml + .github/actions/setup-candidate-sdk/action.yml overwrite-settings: false - name: Configure Maven repositories shell: bash run: | set -euo pipefail + # Reuse third-party dependencies, never a previously installed HugeGraph SDK. + rm -rf -- "$HOME/.m2/repository/org/apache/hugegraph" + rm -f -- "$HOME/.m2/repository/candidate-sdk-manifest.json" maven_version=$(mvn -version | sed $'s/\033\\[[0-9;]*m//g' | sed -n 's/^Apache Maven \([0-9.]*\).*/\1/p' | head -1) if [[ ! "$maven_version" =~ ^([0-9]+)\.([0-9]+)\. ]] || (( 10#${BASH_REMATCH[1]:-0} < 3 || diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index 78e541e28..c43242cd8 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -87,15 +87,6 @@ jobs: with: node-version: '18.20.8' - # we also should cache python & yarn & downloads to avoid useless work - - name: Cache Maven packages - uses: actions/cache@v6 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Cache Node modules uses: actions/cache@v6 with: diff --git a/AGENTS.md b/AGENTS.md index 9f8ca9eb7..abce8c1d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,10 +132,12 @@ hugegraph-loader, hugegraph-tools, hugegraph-hubble, hugegraph-spark-connector ## Docker +Run from the Toolchain repository root; the Dockerfiles copy shared SDK helpers. + ```bash # Loader -cd hugegraph-loader && docker build -t hugegraph/hugegraph-loader:latest . +docker build -f hugegraph-loader/Dockerfile -t hugegraph/hugegraph-loader:latest . # Hubble -cd hugegraph-hubble && docker build -t hugegraph/hugegraph-hubble:latest . +docker build -f hugegraph-hubble/Dockerfile -t hugegraph/hugegraph-hubble:latest . ``` diff --git a/README.md b/README.md index b2bad81a0..a40b242c2 100644 --- a/README.md +++ b/README.md @@ -433,7 +433,7 @@ docker run --rm \ ./bin/hugegraph-loader.sh -f /config/mapping.json ``` -Build images locally: +Build images from the Toolchain repository root so the context contains the shared SDK bootstrap and verifier: ```bash # Loader docker build -f hugegraph-loader/Dockerfile \ diff --git a/docs/ci.md b/docs/ci.md index 9c1770af8..1bc6897e0 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -38,11 +38,11 @@ Each image job builds the checkout's actual Dockerfile, records its image ID and Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement affected-module selection without adding unrelated modules. Their results remain advisory. -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. CI locks the complete immutable source commit (shown as `e62c96` here), so later master updates do not change the build inputs. The SDK action, CI fixtures, image defaults and packaging verifier use the same locked source commit. +The historical Server 1.7 fixture uses the official ASF 1.7.0 binary archive with a pinned official SHA-512 and runs on its own Java 11 JVM. Downloads fail without a source-build fallback. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. CI locks the complete immutable source commit (shown as `e62c96` here), so later master updates do not change the build inputs. The SDK action, CI fixtures, image defaults and packaging verifier use the same locked source commit. -The released Server package and Hubble's candidate fixture are each built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly POM versions before installation, including the root parent, so dependency and license metadata readers consume concrete coordinates without changing the locked source POMs. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. +The official release archive is downloaded once and cached; Hubble's candidate fixture is built once, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly POM versions before installation, including the root parent, so dependency and license metadata readers consume concrete coordinates without changing the locked source POMs. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. -The shared Java initialization action requires Maven 3.9 or newer and supplies explicit settings through `MAVEN_ARGS`, preserving existing arguments and user settings. Dependencies are resolved from Central before ASF Stage, with Stage snapshots disabled; build plugins use the standard public repositories. Maven cache keys include the repository settings so changes to the Stage URL invalidate the cache. +The shared Java initialization action requires Maven 3.9 or newer and supplies explicit settings through `MAVEN_ARGS`, preserving existing arguments and user settings. Dependencies are resolved from Central before ASF Stage, with Stage snapshots disabled; build plugins use the standard public repositories. Maven cache keys include repository settings and SDK preparation inputs. Only third-party dependencies seed the isolated SDK repository; cached HugeGraph artifacts are discarded, and Hubble reuses the shared Maven cache without a second overlapping cache. Loader validates its existing profiles against both configured Server baselines. The Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate NameNode and DataNode containers on the host network. Tests run on Java 17 and use `localhost:8020`. Startup checks live DataNode registration and a real block write/read before testing; containers are removed afterward. Image pulling and readiness are bounded. Cold runners still need to pull the image. diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 5228f618d..c7cedf264 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -195,15 +195,14 @@ Download the toolchain source code. git clone https://github.com/apache/hugegraph-toolchain.git ``` -Compile `hubble`. It depends on the loader and client, so you need to build these dependencies in advance during the compilation process (you can skip this step later). +For the Java 17 candidate, prepare the locked SDK using the [bootstrap instructions](../docs/java17-migration.md#build-the-locked-candidate). Build Client and Loader first, then Hubble with the same isolated Maven repository: ```bash cd hugegraph-toolchain -sudo pip install -r hugegraph-hubble/hubble-dist/assembly/travis/requirements.txt -mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp -cd hugegraph-hubble -mvn -e compile package -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp -cd apache-hugegraph-hubble-* +python3 -m pip install -r hugegraph-hubble/hubble-dist/assembly/travis/requirements.txt +mvn -Dmaven.repo.local="$candidate_dir/m2" install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp +(cd hugegraph-hubble && mvn -Dmaven.repo.local="$candidate_dir/m2" package -Dmaven.javadoc.skip=true -DskipTests -ntp) +cd hugegraph-hubble/apache-hugegraph-hubble-* ``` Run `hubble` @@ -214,6 +213,12 @@ bin/start-hubble.sh -d ### 3. User docker image (Convenient for Test/Dev) +To build the current checkout, run from the Toolchain repository root: + +```bash +docker build -f hugegraph-hubble/Dockerfile -t hugegraph/hugegraph-hubble:latest . +``` + We can use `docker run -itd --name=hubble -p 8088:8088 hugegraph/hubble` to quickly start [hubble](https://hub.docker.com/r/hugegraph/hubble). An you can visit [hubble deploy doc](https://hugegraph.apache.org/docs/quickstart/hugegraph-hubble/#2-deploy) for more details. Then we should follow the [hubble workflow doc](https://hugegraph.apache.org/docs/quickstart/hugegraph-hubble/#3platform-workflow) to create the graph. diff --git a/hugegraph-loader/README.md b/hugegraph-loader/README.md index 83abc79b4..9b660f792 100644 --- a/hugegraph-loader/README.md +++ b/hugegraph-loader/README.md @@ -56,11 +56,10 @@ Install ojdbc8 to the local maven repository, enter the directory where ojdbc8.j mvn install:install-file -Dfile=./ojdbc8.jar -DgroupId=com.oracle -DartifactId=ojdbc8 -Dversion=12.2.0.1 -Dpackaging=jar ``` -Compile and generate tar package: +For the Java 17 candidate, prepare the locked SDK using the [bootstrap instructions](../docs/java17-migration.md#build-the-locked-candidate), then compile from the Toolchain repository root with the same isolated Maven repository: -``` -cd hugegraph-loader -mvn clean package -DskipTests +```bash +mvn -Dmaven.repo.local="$candidate_dir/m2" clean package -pl hugegraph-client,hugegraph-loader -am -DskipTests -ntp ``` ### 2.3 Use docker image (Convenient for Test/Dev) @@ -72,6 +71,12 @@ Use the command `docker run -itd --name loader hugegraph/loader` to start loader If you want to load your data, you can mount the data folder like `-v /path/to/data/file:/loader/file` +To build the current checkout, run from the Toolchain repository root: + +```bash +docker build -f hugegraph-loader/Dockerfile -t hugegraph/hugegraph-loader:latest . +``` + #### 2.3.2 Docker-compose The example `docker-compose.yml` is [here](./docker/example/docker-compose.yml). Use the command `docker-compose up -d` to deploy `loader` with `server` and `hubble`. @@ -121,14 +126,7 @@ docker exec -it loader bin/hugegraph-loader.sh -g hugegraph -f example/file/stru The [loader homepage](https://hugegraph.apache.org/docs/quickstart/hugegraph-loader/) contains more information about it. -The CI server installer `assembly/travis/install-hugegraph-from-source.sh ` -uses `~/hugegraph-cache-` by default. `SERVER_CACHE_DIR` selects the exact cache -directory. A populated cache must contain one server archive and its generated -`server-provenance` file matching the repository, commit, archive name and SHA-256. -Existing custom caches without provenance and mismatched caches are preserved and -rejected; use a fresh directory to build a verified cache, and separate directories for -different server baselines. Loader CI uses a `provenance-v1` cache key so its first run -builds a cache with provenance. The installer does not add SHA subdirectories to `SERVER_CACHE_DIR`. +Server 1.7 compatibility CI uses the pinned official ASF 1.7.0 binary archive. The fixture cache verifies the release version, official source URL, official SHA-512 and cached archive SHA-256 before startup. A failed download does not fall back to rebuilding the release tag. Unpublished candidate SDKs and their Server fixture continue to use the locked ASF source build. ## 5. License From 926c4552311faaf38aef1f9a22fc808e0531a9b0 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 00:30:28 +0800 Subject: [PATCH 33/38] fix(hubble): exclude unused Word runtime keep Server configuration and Struct dependencies leave segmentation execution in the Server process avoid bundling the unused GPL Word dependency --- hugegraph-hubble/hubble-be/pom.xml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hugegraph-hubble/hubble-be/pom.xml b/hugegraph-hubble/hubble-be/pom.xml index 6d4d83e05..646a20476 100644 --- a/hugegraph-hubble/hubble-be/pom.xml +++ b/hugegraph-hubble/hubble-be/pom.xml @@ -264,12 +264,16 @@ com.sun.xml.bind jaxb-impl - + org.apache.hugegraph hg-store-common + + + org.apdplat + word + From 926add00643b3f3ac1c516ecc5c022056c29e773 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 01:37:05 +0800 Subject: [PATCH 34/38] fix(ci): use the ASF release distribution - download Server 1.7 from its current ASF location - retain the official checksum and source validation - update the fixture identity expectation --- .github/actions/setup-hugegraph-server/release.py | 4 ++-- .github/actions/setup-hugegraph-server/runtime_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-hugegraph-server/release.py b/.github/actions/setup-hugegraph-server/release.py index 0e1b7472b..8c812261d 100644 --- a/.github/actions/setup-hugegraph-server/release.py +++ b/.github/actions/setup-hugegraph-server/release.py @@ -27,8 +27,8 @@ REPOSITORY = "apache/hugegraph" COMMIT = "b12425c2032bf0d21a97b8221f42a18055c2982f" ARCHIVE_NAME = "apache-hugegraph-incubating-1.7.0.tar.gz" -SOURCE_URL = "https://archive.apache.org/dist/incubator/hugegraph/1.7.0/" + ARCHIVE_NAME -# Verified against SOURCE_URL + ".sha512" in the official ASF archive. +SOURCE_URL = "https://downloads.apache.org/hugegraph/1.7.0/" + ARCHIVE_NAME +# Verified against SOURCE_URL + ".sha512" in the official ASF distribution. SHA512 = ("ba093203e817f17582895ff10ceb0458498c886ec7a687fd0f3e5f56ba739454" "ec928a592bdad8a3685651dc9f8ef904d082f9530d4aae6a83b5d9d9626ad13f") diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index 2814baab4..bec7aa5fa 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -240,7 +240,7 @@ def test_exact_cache_hit_never_builds_or_starts(self): def test_release_pin_matches_official_identity_and_rejects_wrong_bytes(self): self.assertEqual('1.7.0', self.release.VERSION) - self.assertEqual('https://archive.apache.org/dist/incubator/hugegraph/1.7.0/' + + self.assertEqual('https://downloads.apache.org/hugegraph/1.7.0/' + self.release.ARCHIVE_NAME, self.release.SOURCE_URL) self.assertEqual('apache/hugegraph', self.release.REPOSITORY) self.assertEqual(SHA, self.release.COMMIT) From 5d81f6a205297663faeae08f691c6653dbd1757f Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 02:06:36 +0800 Subject: [PATCH 35/38] fix(ci): locate the Hubble release server - find the unique runnable Server in official packages - preserve the two-instance configuration and tests - remove macOS sidecars before graph discovery --- .../hugegraph-server1/install-hugegraph.sh | 21 ++++++++++++++++--- .../hugegraph-server2/install-hugegraph.sh | 21 ++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh index e22479c22..06c7a4c36 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh +++ b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh @@ -27,9 +27,24 @@ ARCHIVES=(apache-hugegraph-*.tar.gz) TAR=${ARCHIVES[0]} tar -zxvf "$TAR" -C "${SERVER_PARENT_DIR}" >/dev/null 2>&1 -HUGEGRAPH_NAME=${TAR%%.tar*} -SERVER_DIR="${SERVER_PARENT_DIR}"/$HUGEGRAPH_NAME -echo $SERVER_DIR +# Official releases wrap Server alongside PD/Store; candidate archives do not. +# Remove macOS sidecars before graph configuration discovery, as Client CI does. +find "$SERVER_PARENT_DIR" -type f -name '._*' -delete +SERVER_DIRS=() +for dir in "$SERVER_PARENT_DIR"/apache-hugegraph-*/ "$SERVER_PARENT_DIR"/apache-hugegraph-*/*/; do + if [[ -f "$dir/conf/graphs/hugegraph.properties" && + -f "$dir/conf/rest-server.properties" && + -f "$dir/conf/gremlin-server.yaml" && + -x "$dir/bin/init-store.sh" && -x "$dir/bin/start-hugegraph.sh" ]]; then + SERVER_DIRS+=("$dir") + fi +done +if [[ ${#SERVER_DIRS[@]} -ne 1 ]]; then + printf 'Expected exactly one runnable server distribution, found %s\n' "${#SERVER_DIRS[@]}" >&2 + exit 1 +fi +SERVER_DIR=${SERVER_DIRS[0]} +echo "$SERVER_DIR" python3 "${SERVER_CONFIG_DIR}/../configure_gremlin_fixture.py" \ "${SERVER_DIR}" "${SERVER_CONFIG_DIR}/gremlin-server.yaml" \ diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh index 224240b74..f07adf740 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh +++ b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh @@ -27,9 +27,24 @@ ARCHIVES=(apache-hugegraph-*.tar.gz) TAR=${ARCHIVES[0]} tar -zxvf "$TAR" -C "${SERVER_PARENT_DIR}" >/dev/null 2>&1 -HUGEGRAPH_NAME=${TAR%%.tar*} -SERVER_DIR="${SERVER_PARENT_DIR}"/$HUGEGRAPH_NAME -echo $SERVER_DIR +# Official releases wrap Server alongside PD/Store; candidate archives do not. +# Remove macOS sidecars before graph configuration discovery, as Client CI does. +find "$SERVER_PARENT_DIR" -type f -name '._*' -delete +SERVER_DIRS=() +for dir in "$SERVER_PARENT_DIR"/apache-hugegraph-*/ "$SERVER_PARENT_DIR"/apache-hugegraph-*/*/; do + if [[ -f "$dir/conf/graphs/hugegraph.properties" && + -f "$dir/conf/rest-server.properties" && + -f "$dir/conf/gremlin-server.yaml" && + -x "$dir/bin/init-store.sh" && -x "$dir/bin/start-hugegraph.sh" ]]; then + SERVER_DIRS+=("$dir") + fi +done +if [[ ${#SERVER_DIRS[@]} -ne 1 ]]; then + printf 'Expected exactly one runnable server distribution, found %s\n' "${#SERVER_DIRS[@]}" >&2 + exit 1 +fi +SERVER_DIR=${SERVER_DIRS[0]} +echo "$SERVER_DIR" python3 "${SERVER_CONFIG_DIR}/../configure_gremlin_fixture.py" \ "${SERVER_DIR}" "${SERVER_CONFIG_DIR}/gremlin-server.yaml" \ From 2c8b37e7369830397f8aa76973b176543430dd28 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 10:20:00 +0800 Subject: [PATCH 36/38] fix(ci): preserve archived release access - fall back only to the verified official archive - keep canonical cache identity and checksum failures - remove the unreachable short-ref rewrite --- .../actions/setup-hugegraph-server/release.py | 17 +++++++++--- .../setup-hugegraph-server/runtime_test.py | 26 +++++++++++++++++++ .../assembly/travis/install-candidate-sdk.sh | 2 +- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-hugegraph-server/release.py b/.github/actions/setup-hugegraph-server/release.py index 8c812261d..df42792d1 100644 --- a/.github/actions/setup-hugegraph-server/release.py +++ b/.github/actions/setup-hugegraph-server/release.py @@ -28,6 +28,7 @@ COMMIT = "b12425c2032bf0d21a97b8221f42a18055c2982f" ARCHIVE_NAME = "apache-hugegraph-incubating-1.7.0.tar.gz" SOURCE_URL = "https://downloads.apache.org/hugegraph/1.7.0/" + ARCHIVE_NAME +ARCHIVE_URL = "https://archive.apache.org/dist/hugegraph/1.7.0/" + ARCHIVE_NAME # Verified against SOURCE_URL + ".sha512" in the official ASF distribution. SHA512 = ("ba093203e817f17582895ff10ceb0458498c886ec7a687fd0f3e5f56ba739454" "ec928a592bdad8a3685651dc9f8ef904d082f9530d4aae6a83b5d9d9626ad13f") @@ -62,10 +63,18 @@ def download(directory): os.close(descriptor) path = pathlib.Path(temporary) try: - subprocess.run(["curl", "--fail", "--location", "--show-error", "--silent", - "--connect-timeout", "20", "--max-time", "900", "--retry", "2", - "--proto", "=https", "--proto-redir", "=https", - "--output", str(path), SOURCE_URL], check=True) + command = ["curl", "--fail", "--location", "--show-error", "--silent", + "--connect-timeout", "20", "--max-time", "900", "--retry", "2", + "--proto", "=https", "--proto-redir", "=https", + "--output", str(path)] + try: + subprocess.run(command + [SOURCE_URL], check=True) + except subprocess.CalledProcessError: + # Superseded releases leave downloads but remain in the ASF archive. + # The manifest identity stays canonical regardless of transport. + print("ASF distribution download failed; trying the official archive", file=sys.stderr) + subprocess.run(command + [ARCHIVE_URL], check=True) + # A checksum mismatch is fatal; never hide it by trying another source. verify(path) path.replace(directory / "server.tar.gz") finally: diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index bec7aa5fa..024d2b79a 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -295,6 +295,32 @@ def test_release_download_keeps_official_identity_and_never_builds(self): self.assertIn('--proto\n=https', args) self.assertIn(self.release.SOURCE_URL, args) + def test_release_archive_fallback_keeps_canonical_identity(self): + self.command('curl', 'printf "%s\\n" "$@" >> "$HOME/download-args"\n' + 'if [[ ! -f "$HOME/primary-failed" ]]; then\n' + ' touch "$HOME/primary-failed"; exit 22\nfi\n' + 'while [[ $# -gt 0 ]]; do\n' + ' if [[ "$1" == --output ]]; then printf archive > "$2"; exit 0; fi\n' + ' shift\ndone\nexit 1\n') + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertEqual(0, result.returncode, result.stderr) + args = (self.dir / 'download-args').read_text() + self.assertLess(args.index(self.release.SOURCE_URL), args.index(self.release.ARCHIVE_URL)) + manifest = json.loads((Path(self.env['FIXTURE_DIR']) / 'manifest.json').read_text()) + self.assertEqual(self.release.SOURCE_URL, manifest['source_url']) + self.assertEqual(hashlib.sha512(b'archive').hexdigest(), manifest['official_sha512']) + + def test_release_bad_checksum_does_not_try_another_source(self): + self.command('curl', 'printf "%s\\n" "$@" >> "$HOME/download-args"\n' + 'while [[ $# -gt 0 ]]; do\n' + ' if [[ "$1" == --output ]]; then printf tampered > "$2"; exit 0; fi\n' + ' shift\ndone\nexit 1\n') + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertNotEqual(0, result.returncode) + self.assertIn('SHA-512 mismatch', result.stderr) + self.assertNotIn(self.release.ARCHIVE_URL, (self.dir / 'download-args').read_text()) + self.assertFalse((Path(self.env['FIXTURE_DIR']) / 'server.tar.gz').exists()) + def test_release_failure_never_falls_back_to_source_or_publishes_cache(self): self.command('curl', 'echo download >> "$HOME/downloads"; exit 22\n') for name in ('git', 'mvn'): diff --git a/hugegraph-client/assembly/travis/install-candidate-sdk.sh b/hugegraph-client/assembly/travis/install-candidate-sdk.sh index e4eff9a07..4bf788328 100755 --- a/hugegraph-client/assembly/travis/install-candidate-sdk.sh +++ b/hugegraph-client/assembly/travis/install-candidate-sdk.sh @@ -54,7 +54,7 @@ SDK_MODULES+=,hugegraph-store/hg-store-client,hugegraph-struct -Dflatten.mode=resolveCiFriendliesOnly -DupdatePomFile=true \ -DskipTests -Dmaven.javadoc.skip=true -ntp) ACTUAL_COMMIT=$(git -C "$SOURCE_DIR" rev-parse HEAD) -if [[ "$SERVER_FETCH_REF" =~ ^([0-9a-fA-F]{6}|[0-9a-fA-F]{40})$ ]]; then +if [[ "$SERVER_FETCH_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then export SERVER_FETCH_REF=$ACTUAL_COMMIT fi python3 - "$SOURCE_DIR" "$CANDIDATE_REPO" "${SERVER_REPOSITORY:-apache/hugegraph}" \ From 3c5891b4256a5a860d6e3b7e2657890d39b01c24 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 10:18:40 +0800 Subject: [PATCH 37/38] fix(hubble): require safe tar extraction support - detect tarfile extraction filter support before runtime setup - keep safe extraction mandatory without an unfiltered fallback - cover unsupported Python with existing archive unit tests --- .../assembly/travis/run_live_hubble_smoke.py | 17 ++++++++++++++++ .../travis/test_run_live_hubble_smoke.py | 20 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py b/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py index ec264d5f0..13dc3f1e8 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py +++ b/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py @@ -122,7 +122,18 @@ def assert_safe_tar_member(member, work_dir): raise RuntimeError(f"Unsafe tar link outside work dir: {member.name}") +def require_safe_archive_extraction(): + if not callable(getattr(tarfile, "data_filter", None)): + raise RuntimeError( + "Safe archive extraction requires Python with tarfile.data_filter; " + "upgrade to the latest security patch of Python 3.10 or newer " + "(extraction filters first appeared in 3.9.17, 3.10.12 and 3.11.4). " + "Unfiltered extraction is not supported." + ) + + def extract_archive(tarball, work_dir): + require_safe_archive_extraction() with tarfile.open(tarball) as archive: members = archive.getmembers() for member in members: @@ -754,6 +765,12 @@ def main(): "pa")) args = parser.parse_args() + if not args.skip_start: + try: + require_safe_archive_extraction() + except RuntimeError as exc: + parser.error(str(exc)) + hubble_url = args.hubble_url.rstrip("/") server_url = args.server_url.rstrip("/") work_dir = args.work_dir or Path(tempfile.mkdtemp(prefix="hubble-694-live-")) diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py b/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py index 752b682db..a3612c4ff 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py +++ b/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py @@ -33,6 +33,26 @@ class ArchiveExtractionTest(unittest.TestCase): + def test_missing_data_filter_fails_before_opening_archive(self): + with mock.patch.object(tarfile, "data_filter", None, create=True), \ + mock.patch.object(tarfile, "open") as open_archive: + with self.assertRaisesRegex(RuntimeError, "tarfile.data_filter"): + SMOKE.extract_archive(Path("hubble.tar.gz"), Path("runtime")) + open_archive.assert_not_called() + + def test_cli_rejects_unsupported_python_before_runtime_setup(self): + with mock.patch.object(tarfile, "data_filter", None, create=True), \ + mock.patch("sys.argv", [str(SCRIPT), "hubble.tar.gz"]), \ + mock.patch.object(SMOKE, "is_healthy") as is_healthy, \ + mock.patch.object(tempfile, "mkdtemp") as create_work_dir, \ + mock.patch("sys.stderr", new_callable=io.StringIO) as stderr: + with self.assertRaises(SystemExit) as error: + SMOKE.main() + self.assertEqual(2, error.exception.code) + self.assertIn("upgrade to the latest security patch", stderr.getvalue()) + is_healthy.assert_not_called() + create_work_dir.assert_not_called() + def test_sequential_symlink_escape_is_rejected(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) From f917937d6885f302dbe5d5318e8b9ac4674995d9 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 10:57:38 +0800 Subject: [PATCH 38/38] fix(ci): discard cached download failures - remove restored Maven negative-cache markers - retain downloaded third-party dependencies - recheck repositories after transient failures --- .github/actions/setup-java-env/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/setup-java-env/action.yml b/.github/actions/setup-java-env/action.yml index a1e522ab7..f84369f52 100644 --- a/.github/actions/setup-java-env/action.yml +++ b/.github/actions/setup-java-env/action.yml @@ -33,6 +33,10 @@ runs: # Reuse third-party dependencies, never a previously installed HugeGraph SDK. rm -rf -- "$HOME/.m2/repository/org/apache/hugegraph" rm -f -- "$HOME/.m2/repository/candidate-sdk-manifest.json" + # Cached failed downloads must not suppress checking a recovered repository. + if [[ -d "$HOME/.m2/repository" ]]; then + find "$HOME/.m2/repository" -type f -name '*.lastUpdated' -delete + fi maven_version=$(mvn -version | sed $'s/\033\\[[0-9;]*m//g' | sed -n 's/^Apache Maven \([0-9.]*\).*/\1/p' | head -1) if [[ ! "$maven_version" =~ ^([0-9]+)\.([0-9]+)\. ]] || (( 10#${BASH_REMATCH[1]:-0} < 3 ||