commit 2ad60e65792e43b145a11bb89b252c1972c0e6e9 Author: Christophe Vila Date: Sun Jan 25 17:05:38 2026 +0100 Initial commit: CSV to SQLite CLI tool CLI tool to import EDF electricity meter readings from CSV files into a SQLite database. Co-Authored-By: Claude Opus 4.5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..183d7f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/target +.DS_Store +.idea/ +data/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5916c26 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,38 @@ +this project is a tool CLI that can take CSV file and push its content into a sqlite database. + +the CSV files have the following columns: + +- Date de relevé de l'index +- Type d'index +- Index Heures pleines (kWh) +- Index Heures creuses (kWh) + +the sqlite database has been created with the following command: + +CREATE TABLE "electricite" ( +"date" TEXT NOT NULL UNIQUE, +"releveHC" INTEGER NOT NULL, +"releveHP" INTEGER NOT NULL, +"comment" TEXT, +PRIMARY KEY("date") +) + +CSV column "Date de relevé de l'index" has to be mapped with database column "date" +CSV column "Index Heures pleines (kWh)" has to be mapped with database column "releveHP" +CSV column "Index Heures creuses (kWh)" has to be mapped with database column "releveHC" + +the CLI takes 2 parameters: + +- first is the name of the CSV file +- second is the JDBC URL (for example, "jdbc:sqlite:/Users/kriss/NextCloud/Kriss/Dev/Perso/conso/db/conso.db") + +when launched, the tool checks that: + +- CSV file exists on the disk +- CSV file contains exactly the described columns +- Sqlite database is accessible through the JDBC URL +- Ensure there won't be any duplicates inserted in the database (which means if the tool is launched twice in a row, no new lines will be inserted) +- Tell the user how many new lines are going to be inserted in the database, and ask his agreement before proceeding +- Performs the insert in the database + +this project is written in Rust. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..db090d5 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,273 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "cc" +version = "1.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pushEDF" +version = "0.1.0" +dependencies = [ + "anyhow", + "csv", + "encoding_rs", + "rusqlite", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "ryu" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zerocopy" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..60d1270 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "pushEDF" +version = "0.1.0" +edition = "2024" + +[dependencies] +csv = "1.3" +rusqlite = { version = "0.32", features = ["bundled"] } +anyhow = "1.0" +encoding_rs = "0.8" diff --git a/package.json b/package.json new file mode 100644 index 0000000..abc6bcc --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "claude": "^0.1.2" + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..5e6f57f --- /dev/null +++ b/src/main.rs @@ -0,0 +1,233 @@ +use anyhow::{anyhow, bail, Context, Result}; +use csv::ReaderBuilder; +use encoding_rs::ISO_8859_15; +use rusqlite::Connection; +use std::collections::HashSet; +use std::env; +use std::fs; +use std::io::{self, Write}; +use std::path::Path; + +const EXPECTED_COLUMNS: [&str; 4] = [ + "Date de relevé de l'index", + "Type d'index", + "Index Heures pleines (kWh)", + "Index Heures creuses (kWh)", +]; + +#[derive(Debug)] +struct Record { + date: String, + releve_hp: i64, + releve_hc: i64, +} + +fn main() -> Result<()> { + let args: Vec = env::args().collect(); + + if args.len() != 3 { + bail!( + "Usage: {} \nExample: {} data.csv \"jdbc:sqlite:/path/to/conso.db\"", + args[0], + args[0] + ); + } + + let csv_path = &args[1]; + let jdbc_url = &args[2]; + + // Check CSV file exists + if !Path::new(csv_path).exists() { + bail!("CSV file '{}' does not exist", csv_path); + } + + // Parse JDBC URL to get SQLite path + let db_path = parse_jdbc_url(jdbc_url)?; + + // Check database is accessible + let conn = Connection::open(&db_path) + .with_context(|| format!("Failed to open database at '{}'", db_path))?; + + // Verify the table exists + verify_table_exists(&conn)?; + + // Read and validate CSV + let records = read_csv(csv_path)?; + println!("Read {} records from CSV file", records.len()); + + // Get existing dates from database + let existing_dates = get_existing_dates(&conn)?; + + // Filter out duplicates + let new_records: Vec<&Record> = records + .iter() + .filter(|r| !existing_dates.contains(&r.date)) + .collect(); + + if new_records.is_empty() { + println!("No new records to insert. All records already exist in the database."); + return Ok(()); + } + + println!( + "\n{} new record(s) will be inserted into the database:", + new_records.len() + ); + for record in &new_records { + println!( + " - {} (HP: {}, HC: {})", + record.date, record.releve_hp, record.releve_hc + ); + } + + // Ask for user confirmation + print!("\nDo you want to proceed? [y/N] "); + io::stdout().flush()?; + + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + + if !input.trim().eq_ignore_ascii_case("y") { + println!("Operation cancelled."); + return Ok(()); + } + + // Insert records + insert_records(&conn, &new_records)?; + + println!( + "Successfully inserted {} record(s) into the database.", + new_records.len() + ); + + Ok(()) +} + +fn parse_jdbc_url(jdbc_url: &str) -> Result { + // Expected format: jdbc:sqlite:/path/to/database.db + let prefix = "jdbc:sqlite:"; + if !jdbc_url.starts_with(prefix) { + bail!( + "Invalid JDBC URL format. Expected 'jdbc:sqlite:/path/to/db', got '{}'", + jdbc_url + ); + } + + let path = &jdbc_url[prefix.len()..]; + if path.is_empty() { + bail!("Database path in JDBC URL is empty"); + } + + Ok(path.to_string()) +} + +fn verify_table_exists(conn: &Connection) -> Result<()> { + let table_exists: bool = conn.query_row( + "SELECT COUNT(*) > 0 FROM sqlite_master WHERE type='table' AND name='electricite'", + [], + |row| row.get(0), + )?; + + if !table_exists { + bail!("Table 'electricite' does not exist in the database"); + } + + Ok(()) +} + +fn read_csv(path: &str) -> Result> { + // Read file as bytes and convert from ISO-8859-15 to UTF-8 + // (EDF CSV exports are in ISO-8859-15 encoding) + let bytes = fs::read(path).with_context(|| format!("Failed to read CSV file '{}'", path))?; + let (content, _, had_errors) = ISO_8859_15.decode(&bytes); + if had_errors { + bail!("Failed to decode CSV file from ISO-8859-15 encoding"); + } + + let mut reader = ReaderBuilder::new() + .delimiter(b';') + .from_reader(content.as_bytes()); + + // Validate headers + let headers = reader.headers()?.clone(); + let header_vec: Vec<&str> = headers.iter().collect(); + + for expected in &EXPECTED_COLUMNS { + if !header_vec.contains(expected) { + bail!( + "CSV file is missing required column '{}'. Found columns: {:?}", + expected, + header_vec + ); + } + } + + // Find column indices + let date_idx = header_vec + .iter() + .position(|&h| h == "Date de relevé de l'index") + .ok_or_else(|| anyhow!("Column 'Date de relevé de l'index' not found"))?; + let hp_idx = header_vec + .iter() + .position(|&h| h == "Index Heures pleines (kWh)") + .ok_or_else(|| anyhow!("Column 'Index Heures pleines (kWh)' not found"))?; + let hc_idx = header_vec + .iter() + .position(|&h| h == "Index Heures creuses (kWh)") + .ok_or_else(|| anyhow!("Column 'Index Heures creuses (kWh)' not found"))?; + + let mut records = Vec::new(); + + for (line_num, result) in reader.records().enumerate() { + let record = result.with_context(|| format!("Failed to parse CSV line {}", line_num + 2))?; + + let date = record + .get(date_idx) + .ok_or_else(|| anyhow!("Missing date value at line {}", line_num + 2))? + .to_string(); + + let hp_str = record + .get(hp_idx) + .ok_or_else(|| anyhow!("Missing HP value at line {}", line_num + 2))?; + let releve_hp: i64 = hp_str + .parse() + .with_context(|| format!("Invalid HP value '{}' at line {}", hp_str, line_num + 2))?; + + let hc_str = record + .get(hc_idx) + .ok_or_else(|| anyhow!("Missing HC value at line {}", line_num + 2))?; + let releve_hc: i64 = hc_str + .parse() + .with_context(|| format!("Invalid HC value '{}' at line {}", hc_str, line_num + 2))?; + + records.push(Record { + date, + releve_hp, + releve_hc, + }); + } + + Ok(records) +} + +fn get_existing_dates(conn: &Connection) -> Result> { + let mut stmt = conn.prepare("SELECT date FROM electricite")?; + let dates: HashSet = stmt + .query_map([], |row| row.get(0))? + .filter_map(|r| r.ok()) + .collect(); + + Ok(dates) +} + +fn insert_records(conn: &Connection, records: &[&Record]) -> Result<()> { + let mut stmt = + conn.prepare("INSERT INTO electricite (date, releveHC, releveHP) VALUES (?1, ?2, ?3)")?; + + for record in records { + stmt.execute((&record.date, record.releve_hc, record.releve_hp)) + .with_context(|| format!("Failed to insert record for date '{}'", record.date))?; + } + + Ok(()) +}