#!/bin/bash

# Database and table details
DATABASE="user_data"
TABLE="user_records"
ORIGIN="BreachCompilation"
CLICKHOUSE_CLIENT="clickhouse-client --port 19000 --user default --password blackCapybara"

# Function to process a single file
process_file() {
  local file="$1"
  echo "Processing file: $file"

  while IFS=":" read -r username password; do
    if [[ -n "$username" && -n "$password" ]]; then
      escaped_username=$(printf '%s\n' "$username" | sed "s/'/'\\\\''/g")
      escaped_password=$(printf '%s\n' "$password" | sed "s/'/'\\\\''/g")
      insert_command="INSERT INTO ${DATABASE}.${TABLE} (username, password, origin) VALUES ('${escaped_username}', '${escaped_password}', '${ORIGIN}')"
      $CLICKHOUSE_CLIENT --query "$insert_command" || echo "Error inserting record from $file: $username" >&2
    fi
  done < "$file"
}

export -f process_file
export DATABASE TABLE ORIGIN CLICKHOUSE_CLIENT

# Process files in parallel (8 jobs at a time)
find . -type f -print0 | parallel -0 -j8 process_file {}

echo "Finished processing all files."
