Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 1 addition & 10 deletions crates/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion crates/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,13 @@ pretty_assertions = { version = "1.4.1" }
proc-macro2 = { version = "1.0.107" }
quote = { version = "1.0.47" }
rand = { version = "0.10.3" }
rapidhash = { version = "4.5.1" }
regex = { version = "1.13.1" }
reqwest = { version = "0.13.5", features = ["multipart"] }
schemars = { version = "1.2.3", features = ["jiff02", "uuid1"] }
secrecy = { version = "0.10.3", features = ["serde"] }
serde = { version = "1.0.229", features = ["derive"] }
serde_json = { version = "1.0.154" }
sha2 = { version = "0.11.1" }
sha3 = { version = "0.12.0" }
strum = { version = "0.28.0", features = ["derive"] }
syn = { version = "3.0.9", default-features = false, features = [
Expand Down
2 changes: 1 addition & 1 deletion crates/cellnoor-api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ jsonwebtoken.workspace = true
phf.workspace = true
postgres-types.workspace = true
rand.workspace = true
rapidhash.workspace = true
regex.workspace = true
reqwest = { optional = true, workspace = true }
schemars.workspace = true
secrecy.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
sha3.workspace = true
strum.workspace = true
tempfile.workspace = true
Expand Down
2 changes: 1 addition & 1 deletion crates/cellnoor-api/openapi.json

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ fn extract_renamed_metric_from_multi_metrics_row(
struct RawFileRecord {
dataset_id: Id,
path: NonemptyString,
rapidhash_v3_hash: String,
sha256_hash: Vec<u8>,
}

fn create_file_tree(
Expand All @@ -238,7 +238,7 @@ fn create_file_tree(
for RawFileRecord {
dataset_id,
path,
rapidhash_v3_hash,
sha256_hash,
} in raw_files
{
let path_str: &str = path.as_ref();
Expand All @@ -260,7 +260,7 @@ fn create_file_tree(
kind: FileTreeNodeKind::File {
raw_file_link: file_path_to_link(*dataset_id, path),
parsed_contents: parsed_files.remove(path),
rapidhash_v3_hash: rapidhash_v3_hash.to_owned(),
sha256_hash: sha256_hash.clone(),
},
});
}
Expand Down Expand Up @@ -386,7 +386,7 @@ mod tests {
kind: FileTreeNodeKind::File {
raw_file_link: format!("/files/chromium-datasets/{dataset_id}/{full_path}"),
parsed_contents,
rapidhash_v3_hash: "0".to_owned(),
sha256_hash: vec![],
},
};
let dir = |full_path: &str, children| FileTreeNode {
Expand Down Expand Up @@ -436,7 +436,7 @@ mod tests {
.map(|p| RawFileRecord {
dataset_id,
path: p.to_nonempty_string(),
rapidhash_v3_hash: "0".to_owned(),
sha256_hash: vec![],
});

assert_eq!(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::io;
use axum::extract::{Multipart, multipart::Field};
use camino::{Utf8Path, Utf8PathBuf};
use cellnoor_types::chromium_dataset::ChromiumDatasetParsedFileData;
use rapidhash::v3::{DEFAULT_RAPID_SECRETS, RapidStreamHasherV3, rapidhash_v3};
use sha2::{Digest, Sha256};
use tokio::io::AsyncWriteExt;

use crate::handlers::chromium_datasets::{
Expand Down Expand Up @@ -107,16 +107,16 @@ async fn write_html_to_disk(
dest: &Utf8Path,
) -> Result<FileHash, UploadFilesError> {
let mut file = tokio::fs::File::create(dest).await?;
let mut hasher = RapidStreamHasherV3::new(&DEFAULT_RAPID_SECRETS);
let mut hasher = Sha256::default();

while let Some(chunk) = form_field.chunk().await? {
file.write_all(&chunk).await?;
hasher.write(&chunk);
hasher.update(&chunk);
}

file.flush().await?;

Ok(FileHash::new(hasher.finish()))
Ok(FileHash::new(hasher.finalize().0))
}

async fn write_csv_or_json_to_disk(
Expand All @@ -127,18 +127,18 @@ async fn write_csv_or_json_to_disk(
let parsed_file = parse_file(content_type, data)?;
tokio::fs::write(&dest, &data).await?;

let hash = rapidhash_v3(data);
let hash = Sha256::digest(data).0;

Ok((FileHash::new(hash), parsed_file))
}

#[derive(Debug, Clone, postgres_types::ToSql, postgres_types::FromSql)]
#[postgres(transparent)]
pub(super) struct FileHash(String);
pub(super) struct FileHash(Vec<u8>);

impl FileHash {
pub(super) fn new(hash: u64) -> Self {
Self(hash.to_string())
pub(super) fn new(hash: [u8; 32]) -> Self {
Self(hash.to_vec())
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ impl Insert for NewRawFile<'_> {
vec![
("dataset_id", dataset_id),
("path", path),
("rapidhash_v3_hash", hash),
("sha256_hash", hash),
]
}
}
Expand Down Expand Up @@ -187,7 +187,7 @@ pub(in super::super) fn parse_csv(
#[cfg(test)]
mod tests {
use pretty_assertions::assert_eq;
use rapidhash::v3::rapidhash_v3;
use sha2::{Digest, Sha256};
use uuid::Uuid;

use super::{StagedFile, UploadPath, write_files_to_db};
Expand Down Expand Up @@ -218,7 +218,7 @@ mod tests {
))
.unwrap(),
parsed_data: None,
hash: FileHash::new(rapidhash_v3(Uuid::new_v4().as_bytes())),
hash: FileHash::new(Sha256::digest(Uuid::new_v4().as_bytes()).0),
};

let lowercase_path = "per_sample_outs/sample1/summary.csv";
Expand Down
2 changes: 1 addition & 1 deletion crates/cellnoor-types/src/file_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ pub struct FileTreeNode<T> {
pub enum FileTreeNodeKind<T> {
File {
raw_file_link: String,
rapidhash_v3_hash: String,
sha256_hash: Vec<u8>,
parsed_contents: Option<T>,
},
Directory {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
alter table chromium_dataset_raw_file drop rapidhash_v3_hash;

alter table chromium_dataset_raw_file add sha256_hash bytea not null;

alter table chromium_dataset_raw_file add unique (dataset_id, sha256_hash);
2 changes: 1 addition & 1 deletion packages/cellnoor-client/cellnoor-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7684,8 +7684,8 @@ export interface components {
path: string;
} & ({
parsed_contents?: components["schemas"]["ChromiumDatasetParsedFileData"] | null;
rapidhash_v3_hash: string;
raw_file_link: string;
sha256_hash: number[];
/** @constant */
type: "file";
} | {
Expand Down
Loading