Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions hugegraph-loader/assembly/static/bin/hugegraph-flinkcdc-loader.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,30 @@ fi

# Without a separator all arguments belong to Loader. To set Flink options:
# hugegraph-flinkcdc-loader.sh -p 1 -- --file mapping.json --graph hugegraph
ENGINE_PARAMS=()
HUGEGRAPH_PARAMS=("$@")
FLINK_ARGS=()
LOADER_ARGS=("$@")
for arg in "$@"; do
if [[ "$arg" == "--" ]]; then
HUGEGRAPH_PARAMS=()
LOADER_ARGS=()
while [[ "$1" != "--" ]]; do
ENGINE_PARAMS+=("$1")
FLINK_ARGS+=("$1")
shift
done
shift
HUGEGRAPH_PARAMS=("$@")
LOADER_ARGS=("$@")
break
fi
done

# Validate Loader options without rewriting their original argument boundaries.
(
# The shared parser permits empty engine arrays on Bash 3.
set +u
source "$BIN_DIR/get-params.sh"
get_params "${LOADER_ARGS[@]}"
)

# Do not echo arguments: they can contain authentication credentials.
exec "$FLINK_HOME/bin/flink" run ${ENGINE_PARAMS[@]+"${ENGINE_PARAMS[@]}"} \
exec "$FLINK_HOME/bin/flink" run ${FLINK_ARGS[@]+"${FLINK_ARGS[@]}"} \
-c org.apache.hugegraph.loader.flink.HugeGraphFlinkCDCLoader \
"${ASSEMBLY_JARS[0]}" ${HUGEGRAPH_PARAMS[@]+"${HUGEGRAPH_PARAMS[@]}"}
"${ASSEMBLY_JARS[0]}" ${LOADER_ARGS[@]+"${LOADER_ARGS[@]}"}
1 change: 1 addition & 0 deletions hugegraph-loader/docs/flink-cdc-java17.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ bin/hugegraph-flinkcdc-loader.sh --file mapping.json --graph hugegraph
```

Place Flink options before `--` and Loader options after it. Paths with spaces remain individual arguments; the launcher does not print argument values.
Missing values for recognized Loader options, such as `--password` or `--file`, are rejected before invoking Flink.

```bash
bin/hugegraph-flinkcdc-loader.sh -p 1 -- \
Expand Down
14 changes: 10 additions & 4 deletions hugegraph-loader/src/test/scripts/flink-launcher.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const { mkdtempSync, mkdirSync, copyFileSync, writeFileSync, readFileSync, realpathSync, rmSync } = require('node:fs');
const { mkdtempSync, mkdirSync, copyFileSync, writeFileSync, readFileSync, realpathSync, rmSync, existsSync } = require('node:fs');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');

Expand All @@ -30,6 +30,7 @@ test('CDC launcher keeps argument boundaries and does not expose arguments', t =
mkdirSync(join(root, 'flink/bin'), { recursive: true });
const script = join(root, 'loader/bin/hugegraph-flinkcdc-loader.sh');
copyFileSync(resolve(__dirname, '../../../assembly/static/bin/hugegraph-flinkcdc-loader.sh'), script);
copyFileSync(resolve(__dirname, '../../../assembly/static/bin/get-params.sh'), join(root, 'loader/bin/get-params.sh'));
const argsFile = join(root, 'args');
writeFileSync(join(root, 'flink/bin/flink'), '#!/bin/bash\nprintf "%s\\0" "$@" > "$PROBE_ARGS"\n' +
'exit "${PROBE_EXIT_CODE:-0}"\n', { mode: 0o755 });
Expand All @@ -52,9 +53,14 @@ test('CDC launcher keeps argument boundaries and does not expose arguments', t =
assert.deepEqual(readFileSync(argsFile, 'utf8').split('\0').slice(0, -1),
['run', '-c', 'org.apache.hugegraph.loader.flink.HugeGraphFlinkCDCLoader', jar,
'--file', 'mapping.json', '--graph', 'fixture']);
const rejected = run(['--file'], { PROBE_EXIT_CODE: '2' });
assert.equal(rejected.status, 2, rejected.stderr);
assert.equal(rejected.stdout, '');
for (const option of ['--password', '--file']) {
rmSync(argsFile, { force: true });
const rejected = run([option]);
assert.equal(rejected.status, 2, `${option}: ${rejected.stderr}`);
assert.match(rejected.stderr, new RegExp(`Missing value for ${option}`));
assert.equal(rejected.stdout, '');
assert.equal(existsSync(argsFile), false, 'Flink must not run for a missing Loader option value');
}
assert.notEqual(run([], { FLINK_HOME: '' }).status, 0);
writeFileSync(join(root, 'loader/lib/apache-hugegraph-loader-extra-shaded.jar'), 'fixture');
assert.notEqual(run([]).status, 0);
Expand Down
Loading