Forem: LS

sql python note

LS — Thu, 04 Dec 2025 14:32:27 +0000

flyway implementation

LS — Wed, 26 Nov 2025 17:23:28 +0000

#!/usr/bin/env python3
"""
Generate DML SQL for populating the etl_config table from a YAML file,
and write it as a Flyway migration.

Usage:
    python etl_config.py etl_config --name <NAME> [--yaml-file PATH] [--table etl_config]

Assumptions:
    - YAML structure (recommended):

        etl_config:
          - pipeline_name: accounts_daily
            enabled: true
            schedule_cron: "0 3 * * *"
            source_table: accounts_raw
            target_table: accounts_curated
          - ...

    - The script is stored somewhere under the repo root; update CONFIG_DIR and
      MIGRATIONS_DIR below to match your layout.
"""

import argparse
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Sequence

import yaml  # pip install pyyaml


# ---------------------------------------------------------------------------
# CONFIGURE THESE PATHS FOR YOUR REPO LAYOUT
# ---------------------------------------------------------------------------

# Repo root: adjust the number of `.parents[...]` if needed for your layout.
REPO_ROOT = Path(__file__).resolve().parents[2]

# Where YAML config files live (one per --name).
# e.g. infrastructure/flyway/data-domains/accounts_datastore/etl_config/config/<name>.yaml
CONFIG_DIR = (
    REPO_ROOT
    / "infrastructure"
    / "flyway"
    / "data-domains"
    / "accounts_datastore"
    / "etl_config"
    / "config"
)

# Where Flyway looks for migrations for this domain.
# e.g. infrastructure/flyway/data-domains/accounts_datastore/etl_config/migrations/
MIGRATIONS_DIR = (
    REPO_ROOT
    / "infrastructure"
    / "flyway"
    / "data-domains"
    / "accounts_datastore"
    / "etl_config"
    / "migrations"
)


# ---------------------------------------------------------------------------
# HELPERS
# ---------------------------------------------------------------------------

def sql_literal(value: Any) -> str:
    """
    Convert a Python value into a SQL literal string.

    - None -> NULL
    - bool -> true/false
    - numbers -> as-is
    - everything else -> single-quoted string with quotes escaped
    """
    if value is None:
        return "NULL"

    if isinstance(value, bool):
        return "true" if value else "false"

    if isinstance(value, (int, float)):
        return str(value)

    # treat everything else as string
    text = str(value)
    # escape single quotes by doubling them
    text = text.replace("'", "''")
    return f"'{text}'"


def load_yaml(path: Path) -> Dict[str, Any]:
    if not path.exists():
        raise FileNotFoundError(f"YAML config not found: {path}")
    with path.open("r", encoding="utf-8") as f:
        data = yaml.safe_load(f)
    if data is None:
        raise ValueError(f"YAML file is empty: {path}")
    return data


def extract_records(yaml_data: Dict[str, Any]) -> List[Dict[str, Any]]:
    """
    Try to get the list of records from YAML.

    Preferred: top-level key "etl_config" is a list.
    Fallback: YAML itself is a list.
    """
    if isinstance(yaml_data, list):
        if not yaml_data:
            raise ValueError("YAML list is empty; nothing to generate.")
        return yaml_data

    if "etl_config" in yaml_data:
        records = yaml_data["etl_config"]
        if not isinstance(records, list):
            raise ValueError("Expected 'etl_config' key to contain a list of records.")
        if not records:
            raise ValueError("'etl_config' list is empty; nothing to generate.")
        return records

    raise ValueError("YAML must be a list or contain an 'etl_config' list.")


def infer_columns(records: Sequence[Dict[str, Any]]) -> List[str]:
    """
    Infer the list of columns from the first record and ensure all records
    have exactly the same keys. This keeps the SQL deterministic.
    """
    first_keys = set(records[0].keys())
    if not first_keys:
        raise ValueError("First record has no columns; cannot infer schema.")

    for idx, rec in enumerate(records[1:], start=2):
        keys = set(rec.keys())
        if keys != first_keys:
            raise ValueError(
                f"Record #{idx} has different keys.\n"
                f"Expected: {sorted(first_keys)}\n"
                f"Found   : {sorted(keys)}"
            )

    return sorted(first_keys)


def build_insert_sql(table: str, records: Sequence[Dict[str, Any]]) -> str:
    """
    Build a SQL script with INSERTs for all records.
    """
    columns = infer_columns(records)
    col_list = ", ".join(columns)

    statements: List[str] = []

    for rec in records:
        values_sql = ", ".join(sql_literal(rec.get(col)) for col in columns)
        stmt = f"INSERT INTO {table} ({col_list}) VALUES ({values_sql});"
        statements.append(stmt)

    script = "BEGIN;\n\n" + "\n\n".join(statements) + "\n\nCOMMIT;\n"
    return script


def next_migration_filename(config_type: str, name: str) -> str:
    """
    Produce a Flyway-style migration filename, e.g.:

        V20251127_123456__etl_config_accounts_daily.sql
    """
    ts = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
    safe_type = config_type.replace("-", "_")
    safe_name = name.replace("-", "_")
    return f"V{ts}__{safe_type}_{safe_name}.sql"


def write_migration(content: str, filename: str) -> Path:
    MIGRATIONS_DIR.mkdir(parents=True, exist_ok=True)
    out_path = MIGRATIONS_DIR / filename
    out_path.write_text(content, encoding="utf-8")
    return out_path


# ---------------------------------------------------------------------------
# MAIN
# ---------------------------------------------------------------------------

def parse_args(argv: Sequence[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Generate Flyway DML migration for etl_config from YAML."
    )
    parser.add_argument(
        "config_type",
        choices=["etl_config"],
        help="Type of config to generate (currently only 'etl_config').",
    )
    parser.add_argument(
        "--name",
        required=True,
        help="Logical name for this config; used to pick YAML and to name the migration.",
    )
    parser.add_argument(
        "--yaml-file",
        help="Optional explicit path to YAML file. "
             "If omitted, defaults to CONFIG_DIR/<name>.yaml",
    )
    parser.add_argument(
        "--table",
        default="etl_config",
        help="Target database table name (default: etl_config).",
    )
    return parser.parse_args(argv)


def main(argv: Sequence[str]) -> int:
    args = parse_args(argv)

    # Determine which YAML file to load
    if args.yaml_file:
        yaml_path = Path(args.yaml_file).resolve()
    else:
        yaml_path = (CONFIG_DIR / f"{args.name}.yaml").resolve()

    print(f"Using YAML config: {yaml_path}")

    try:
        yaml_data = load_yaml(yaml_path)
        records = extract_records(yaml_data)
        sql_script = build_insert_sql(args.table, records)
    except Exception as exc:
        print(f"ERROR: {exc}", file=sys.stderr)
        return 1

    filename = next_migration_filename(args.config_type, args.name)
    out_path = write_migration(sql_script, filename)

    print(f"Generated migration: {out_path}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main(sys.argv[1:]))

merge jsonschema

LS — Tue, 28 Oct 2025 12:39:11 +0000

@SuppressWarnings("unchecked")
private static Document innerJsonSchema(Document full) {
  // full is { "$jsonSchema": { ... } }
  return (Document) full.get("$jsonSchema");
}

private static Document mergeSchemasAsOneOf(Document... fullSchemas) {
  List<Document> versions = new ArrayList<>();
  for (Document full : fullSchemas) {
    versions.add(innerJsonSchema(full));
  }
  return new Document("$jsonSchema", new Document("oneOf", versions));
}

// If you prefer "anyOf":
private static Document mergeSchemasAsAnyOf(Document... fullSchemas) {
  List<Document> versions = new ArrayList<>();
  for (Document full : fullSchemas) {
    versions.add(innerJsonSchema(full));
  }
  return new Document("$jsonSchema", new Document("anyOf", versions));
}

import org.bson.Document;
import org.bson.json.JsonMode;
import org.bson.json.JsonWriterSettings;

import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;

public final class SchemaDumper {
  private SchemaDumper() {}

  /** Dump the WHOLE validator doc: { "$jsonSchema": { ... } } */
  public static Path dumpValidator(String filenameWithoutExt, Document validator, Path outDir) throws Exception {
    Files.createDirectories(outDir);
    JsonWriterSettings pretty = JsonWriterSettings.builder()
        .indent(true)
        .outputMode(JsonMode.RELAXED)
        .build();
    String json = validator.toJson(pretty);
    Path file = outDir.resolve(filenameWithoutExt + ".json");
    Files.writeString(file, json, StandardCharsets.UTF_8);
    return file;
  }

  /** Dump only the INNER $jsonSchema object (no wrapper). */
  public static Path dumpInnerSchema(String filenameWithoutExt, Document validator, Path outDir) throws Exception {
    Document inner = validator.get("$jsonSchema", Document.class);
    if (inner == null) throw new IllegalArgumentException("Validator missing $jsonSchema");
    return dumpValidator(filenameWithoutExt, new Document("$jsonSchema", inner), outDir);
  }
}


Document cd18  = loadJsonSchema(CreditDecisioning.class, "{}");
Document cd20  = loadJsonSchema(CreditDecisioningV2.class, "{}");
Document merged = mergeSchemasAsAnyOf(cd18, cd20); // or oneOf

Path out = Path.of("target", "schemas");
SchemaDumper.dumpValidator("creditDecisioning-merged", merged, out);
// or, if someone wants only the inside object:
SchemaDumper.dumpInnerSchema("creditDecisioning-merged-inner", merged, out);

date conversion angular

LS — Wed, 22 Oct 2025 13:25:59 +0000

What it’s doing (and why it shifts the date)
const dateObj = new Date(rawDateValue);                // if 'YYYY-MM-DD' → parsed as **UTC midnight**
const utcDate = new Date(Date.UTC(
  dateObj.getFullYear(), dateObj.getMonth(), dateObj.getDate()
));
const dateOnly = utcDate.toISOString().slice(0, 10);   // ISO in **UTC**
controlWithMetadata.control.setValue(dateOnly);        // now the control holds a **string**


If rawDateValue is 'YYYY-MM-DD', new Date('YYYY-MM-DD') is interpreted as UTC.

You then build another UTC date and call .toISOString() (still UTC).

Later, something (datepicker / form rebuild) re-parses that 'YYYY-MM-DD' string back to a Date. In JS, that string is again treated as UTC, but displayed in local time, so in US timezones it becomes the previous day.

Toggling edit mode runs this code again, compounding the effect → you observe -1 day (sometimes -2 around DST / multiple conversions).

Also, you’re changing the control type from a Date to a string, which makes the datepicker/serialization codepath even more fragile.

Minimal, targeted fix

Keep the control value as a Date (local midnight), never as a string; avoid toISOString() and avoid new Date('YYYY-MM-DD').

// Replace your cleanupDates implementation with this:
cleanupDates(taskAreaMember: TaskAreaMember) {
  if (taskAreaMember.fieldType !== 'date') return;

  taskAreaMember.controls.forEach((controlWithMetadata) => {
    const v = controlWithMetadata.control.value;
    if (!v) return;

    let d: Date;

    if (v instanceof Date) {
      // Normalize to local midnight, preserving Y/M/D only.
      d = new Date(v.getFullYear(), v.getMonth(), v.getDate());
    } else if (typeof v === 'string') {
      // Expecting 'YYYY-MM-DD' → construct **local** date (NOT new Date(ymd))
      const [y, m, dStr] = v.split('-').map(Number);
      d = new Date(y, (m || 1) - 1, dStr || 1); // local midnight
    } else {
      return;
    }

    controlWithMetadata.control.setValue(d, { emitEvent: false });
  });
}

And when you build the payload (save/merge path)

Convert from Date → 'YYYY-MM-DD' explicitly, without toISOString():

function toYmd(date: Date): string {
  const y = date.getFullYear();
  const m = String(date.getMonth() + 1).padStart(2, '0');
  const d = String(date.getDate()).padStart(2, '0');
  return `${y}-${m}-${d}`;
}

// wherever you currently read control.value for birth date:
const val = control.value as Date;
payload.birthDate = toYmd(val);

If you prefer Moment/Material UTC adapter

You can also standardize with Moment’s adapter and useUtc: true, but the core rule remains:

Never call new Date('YYYY-MM-DD')

Never use .toISOString() for birthdays or date-only values

Keep the form control as a Date (or a Moment) and serialize to 'YYYY-MM-DD' only at save time.

Why this will stop the drift

Local-midnight Date(y, m, d) has no timezone surprises when displayed.

You aren’t converting to ISO/UTC and back anymore.

The control type stays consistent (Date ↔ Date), so re-entering edit mode won’t re-interpret a string as UTC.

If you show me the exact save path where you read the birthdate control, I’ll give you the precise 2–3 line diff to swap in toYmd(...) there too.

How to use crypto library in Nodejs

LS — Wed, 24 Sep 2025 13:11:49 +0000

#!/usr/bin/env node
/**
 * secret-encode.js
 *
 * Usage:
 *   node secret-encode.js encrypt "my secret text" myPassword
 *   node secret-encode.js encrypt-file ./input.txt myPassword
 *   node secret-encode.js decrypt <base64-string> myPassword
 *   node secret-encode.js decrypt-file <base64-string> myPassword ./out.txt
 *
 * Output format (JSON string):
 * {
 *   "v": 1,                 // version
 *   "salt": "<base64>",     // PBKDF2 salt
 *   "iv": "<base64>",       // AES-GCM IV
 *   "tag": "<base64>",      // AES-GCM auth tag
 *   "ct": "<base64>"        // ciphertext
 * }
 *
 * This is reversible: keep the password secret and share the encoded JSON string safely.
 */

const crypto = require('crypto');
const fs = require('fs').promises;
const path = require('path');

const PBKDF2_ITER = 120_000; // iterations (adjust upward if you want more CPU cost)
const KEYLEN = 32; // 256-bit key
const SALT_LEN = 16;
const IV_LEN = 12; // recommended for GCM
const AUTH_TAG_LEN = 16;

function deriveKey(password, salt) {
  return crypto.pbkdf2Sync(password, salt, PBKDF2_ITER, KEYLEN, 'sha256');
}

function encryptBuffer(plainBuf, password) {
  const salt = crypto.randomBytes(SALT_LEN);
  const key = deriveKey(password, salt);
  const iv = crypto.randomBytes(IV_LEN);

  const cipher = crypto.createCipheriv('aes-256-gcm', key, iv, { authTagLength: AUTH_TAG_LEN });
  const ct = Buffer.concat([cipher.update(plainBuf), cipher.final()]);
  const tag = cipher.getAuthTag();

  const out = {
    v: 1,
    salt: salt.toString('base64'),
    iv: iv.toString('base64'),
    tag: tag.toString('base64'),
    ct: ct.toString('base64'),
  };
  return JSON.stringify(out);
}

function decryptString(encryptedJsonStr, password) {
  let obj;
  try {
    obj = JSON.parse(encryptedJsonStr);
  } catch (e) {
    throw new Error('Input is not valid JSON. Make sure you pass the encoded JSON string.');
  }
  if (obj.v !== 1) throw new Error('Unsupported version');

  const salt = Buffer.from(obj.salt, 'base64');
  const iv = Buffer.from(obj.iv, 'base64');
  const tag = Buffer.from(obj.tag, 'base64');
  const ct = Buffer.from(obj.ct, 'base64');

  const key = deriveKey(password, salt);
  const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv, { authTagLength: AUTH_TAG_LEN });
  decipher.setAuthTag(tag);

  const plain = Buffer.concat([decipher.update(ct), decipher.final()]);
  return plain;
}

async function run() {
  const args = process.argv.slice(2);
  if (args.length < 1) return printHelp();

  const cmd = args[0];

  try {
    if (cmd === 'encrypt') {
      if (args.length < 3) return printHelp();
      const text = args[1];
      const password = args[2];
      const out = encryptBuffer(Buffer.from(text, 'utf8'), password);
      console.log(out);
      return;
    }

    if (cmd === 'encrypt-file') {
      if (args.length < 3) return printHelp();
      const file = args[1];
      const password = args[2];
      const b = await fs.readFile(file);
      const out = encryptBuffer(b, password);
      console.log(out);
      return;
    }

    if (cmd === 'decrypt') {
      if (args.length < 3) return printHelp();
      const encoded = args[1];
      const password = args[2];
      const plainBuf = decryptString(encoded, password);
      console.log(plainBuf.toString('utf8'));
      return;
    }

    if (cmd === 'decrypt-file') {
      if (args.length < 4) return printHelp();
      const encoded = args[1];
      const password = args[2];
      const outFile = args[3];
      const plainBuf = decryptString(encoded, password);
      await fs.writeFile(outFile, plainBuf);
      console.log(`Wrote decrypted content to ${outFile}`);
      return;
    }

    printHelp();
  } catch (err) {
    console.error('Error:', err.message || err);
    process.exitCode = 2;
  }
}

function printHelp() {
  const me = path.basename(process.argv[1]);
  console.log(`
Usage:
  node ${me} encrypt "my secret text" myPassword
  node ${me} encrypt-file ./input.txt myPassword
  node ${me} decrypt '<encoded-json-string>' myPassword
  node ${me} decrypt-file '<encoded-json-string>' myPassword ./out.txt

Notes:
 - The output is a JSON string (salt, iv, tag, ciphertext in base64). Share that string publicly if you like.
 - NEVER share the password publicly. Send the password via a separate secure channel (phone, Signal, etc).
 - For short secrets you can use "encrypt", for files use "encrypt-file".
`);
}

if (require.main === module) run();

and how to decode

node decrypt-encrypt.js decrypt '{"v":1,"salt":"YkgTYa8bQeuudUVyc6JzTg==","iv":"gYGLAUZFttrwNOHB","tag":"OhQSFgCQuR5iKMQnIh7Jgg==","ct":"43rxWQfb40QNZEEw01EXvDwAH/aRPz7ln6ZgBqah1vIYZGAxkKW4B/xzH/h3u7DEsGMg3kGwZZXTbNp3A/FEiyQhH9gWstmAwy7klJ+hIttCSBVR40WJR6zIKNjVbZDYeDSxsO0HqNGaY7E2gC/ampLwoefl3hHDAeyddsaSk9/4lwji0SYMjrlvFOcHZtMl9CDQiUzkpY7F8zH3TEsu2e8nZuLJojrz3+UEXTdMguGP3hU1YZEAiNfkdW27UiTfgDkHljxm7ZxLKgoe1DBvmnWKIFShixZ2ps1wBYdp6vOyrVv1mXPX3svYSXK5P+f/7WwiuH5QaaVrj1bGSgRmTuT9tfECmToABo9PCi9QoB0vKoRSA/zztTTcG9jSOrkk5GgCmS84r3lBNcZcNyJZ/Iy1IVv8dGJt0r9QjUjPdpGDf1NUTpbrEkUHXdnPpTw8EQNj1mrI88OldqX/xwjayV/2C0rLlS0tRKQCZ0sXSJPBomKL1uPbLGDW8sM6F0iNuMJHoXL6Hd+DvJnmaexDhnW4uHdlwkfM93B9Wqtlzg3wCB80ze1XsQLJml6uP63kbt6rlNG0il1HtuMDOn58oZYAmlciCZ+a/do5dbs7TuxLuOZjVEvh2D8vVA8DCAKT47r0QIaDcrBY8s15+bWJSTnTSVhIXCRs3L3UjINsbNGnvjQjFLpFLjl6El/gGr+9CNyNkmQ3mVEDtZtU8q+0ub6dPawthWzYwsdU7VFC8D04xsfmUBub7QZmWyWPM5BGSTfHITKCtNRwaQvgTyztDQ0fJY8DsK3+ShtdlxBrZY3i2O6PuKL6rEPBxIyqPx4faMwSmBKrvqAYJBgUTdMq6zQVJqugeqVJ7MCEwPDbFW2yUps4IjAqPQhcOQmQ7euSyW/MN3IRzjuqR6iVcv7axCmROK4NAuCNkDpbHHo3rAiJ8WDg1ByMqcM0yKuHUi3E="}' "your secret key"

mongodb shell script counting

LS — Sun, 21 Sep 2025 00:53:37 +0000

// ======= CONFIG: list your base collections here =======
const targetCollections = [
];

// ======= UTILITIES =======
function collExists(name) {
  return db.getCollectionInfos({ name }).length > 0;
}
function countOrZero(name) {
  if (!collExists(name)) return 0;
  // Accurate count; if you prefer faster (approx) use estimatedDocumentCount()
  return db.getCollection(name).countDocuments({});
}
function pad(str, len) {
  const s = String(str);
  return s + " ".repeat(Math.max(0, len - s.length));
}

// ======= RUN =======
const header =
  "Collection".padEnd(28) +
  "original".padEnd(12) +
  "duplicated_".padEnd(14) +
  "backup_".padEnd(12) +
  "backup_new_apps_".padEnd(18);

print(header);
print("-".repeat(header.length));

let totalOriginal = 0;
let totalDuplicated = 0;
let totalBackup = 0;
let totalBackupNewApps = 0;

for (const base of targetCollections) {
  const originalName = base;
  const duplicatedName = `duplicated_${base}`;
  const backupName = `backup_${base}`;
  const backupNewAppsName = `backup_new_apps_${base}`;

  const cOriginal = countOrZero(originalName);
  const cDup = countOrZero(duplicatedName);
  const cBackup = countOrZero(backupName);
  const cBackupNew = countOrZero(backupNewAppsName);

  totalOriginal += cOriginal;
  totalDuplicated += cDup;
  totalBackup += cBackup;
  totalBackupNewApps += cBackupNew;

  print(
    pad(base, 28) +
    pad(cOriginal, 12) +
    pad(cDup, 14) +
    pad(cBackup, 12) +
    pad(cBackupNew, 18)
  );
}

print("-".repeat(header.length));
print(
  pad("TOTAL", 28) +
  pad(totalOriginal, 12) +
  pad(totalDuplicated, 14) +
  pad(totalBackup, 12) +
  pad(totalBackupNewApps, 18)
);

db.application.countDocuments({
$or: [
{ "applicants.0.firstName": { $exists: false } },
{ "applicants.0.firstName": null },
{ "applicants.0.firstName": "" },
{ "applicants.0.lastName": { $exists: false } },
{ "applicants.0.lastName": null },
{ "applicants.0.lastName": "" }
]
});

NodeJs Mongo script query

LS — Sat, 20 Sep 2025 21:12:09 +0000

// findAppIdsCreatedSinceFriday.js
// Usage: MONGO_URI="mongodb://..." node findAppIdsCreatedSinceFriday.js

const { MongoClient } = require("mongodb");

(async () => {
  const uri = process.env.MONGO_URI || "mongodb://localhost:27017";
  const dbName = process.env.DB_NAME
  const collName = process.env.COLL_NAME;

  // 🔒 Hardcode the Friday midnight you want
  // If midnight in New York was intended and you're in EDT (UTC-4),
  // use the equivalent UTC time (04:00:00Z).
  // Example: Sep 19, 2025 00:00 in NYC -> 2025-09-19T04:00:00Z
  const FRIDAY_ISO_UTC = "2025-09-19T04:00:00Z";

  const boundary = new Date(FRIDAY_ISO_UTC);         // JS Date
  const boundaryISO = boundary.toISOString();        // For string comparisons

  const client = new MongoClient(uri);

  try {
    await client.connect();
    const coll = client.db(dbName).collection(collName);

    console.log("Boundary (UTC):", boundaryISO);

    // ✅ Works for both BSON Date and ISO string fields, no $convert/onError
    const filter = {
      $or: [
        // Case 1: field is a BSON Date
        { "createdTimeStamp.dateTime": { $gte: boundary } },

        // Case 2: field is an ISO string; lexicographic compare is valid for ISO-8601
        {
          $and: [
            { "createdTimeStamp.dateTime": { $type: "string" } },
            { "createdTimeStamp.dateTime": { $gte: boundaryISO } }
          ]
        }
      ]
    };

    const projection = { _id: 0, applicationId: 1 };

    const cursor = coll.find(filter, { projection }).batchSize(1000);

    let count = 0;
    for await (const doc of cursor) {
      if (doc && doc.applicationId != null) {
        console.log(String(doc.applicationId));
        count++;
      }
    }
    console.log("Matched:", count);
  } catch (err) {
    console.error("Error:", err);
  } finally {
    await client.close();
  }
})();

Mongo db index creation

LS — Sat, 20 Sep 2025 03:01:04 +0000

// mongosh

function getLastFridayNYMidnightUTC() {
  const now = new Date();
  const daysSinceFriday = (now.getDay() - 5 + 7) % 7; // Friday = 5
  return new Date(now.getFullYear(), now.getMonth(), now.getDate() - daysSinceFriday, 0, 0, 0, 0);
}

const lastFriday = getLastFridayNYMidnightUTC();
print("Last Friday (NY midnight) =", lastFriday.toISOString());

// Find all docs where dateTime > lastFriday
db.getCollection("app").find(
  {
    $expr: {
      $gte: [
        { $dateFromString: { dateString: "$timestamp.dateTime" } },
        lastFriday
      ]
    }
  },
  { _id: 1, "dateTime": 1 }
).forEach(printjson);

Mongo db index creation

LS — Tue, 16 Sep 2025 19:43:54 +0000

How to handle changes in angular tests

LS — Tue, 09 Sep 2025 15:47:22 +0000

Handling Change in Angular Tests (Without Losing Your Mind)

Code changes. Requirements shift. Components get refactored. If your Angular tests shatter every time, they’re testing implementation details—not behavior. Here’s a practical, battle-tested guide to make your spec files easy to update, hard to break, and quick to understand.

1) Adopt a “Public-Surface First” mindset

Test inputs, outputs, and rendered behavior, not internals.

Prefer: “Given these @Input()s, it renders X; when I click Y, it emits Z.”
Avoid: “It calls private method A and sets private property B.”

it('emits value on Save', () => {
  const spy = jest.fn();
  component.save.subscribe(spy);
  component.form.setValue({ name: 'Ada' });

  click(fixture, 'save-btn'); // helper that finds data-testid="save-btn" and clicks
  expect(spy).toHaveBeenCalledWith({ name: 'Ada' });
});

Payoff: Refactors of internals don’t break tests.

2) Stabilize DOM queries with test IDs (or Harnesses)

DOM structures shift during refactors. CSS classes change. Your tests shouldn’t care.

Add stable attributes: data-testid="save-btn".
For Angular Material, prefer Component Test Harnesses—they’re built for resilient querying.

<button data-testid="save-btn" (click)="onSave()">Save</button>

function getEl(fixture: ComponentFixture<any>, id: string): HTMLElement {
  return fixture.nativeElement.querySelector(`[data-testid="${id}"]`);
}

Payoff: Renaming CSS or adding wrappers doesn’t break tests.

3) Use Page Objects for complex components

Page Objects centralize UI knowledge and keep specs readable.

class ProfilePage {
  constructor(private readonly fix: ComponentFixture<ProfileComponent>) {}

  nameInput = () => this.q<HTMLInputElement>('name-input');
  save = () => this.q<HTMLButtonElement>('save-btn');
  setName(v: string) { this.nameInput().value = v; this.nameInput().dispatchEvent(new Event('input')); this.fix.detectChanges(); }
  clickSave() { this.save().click(); this.fix.detectChanges(); }

  private q<T = HTMLElement>(id: string) {
    return this.fix.nativeElement.querySelector(`[data-testid="${id}"]`) as T;
  }
}

Payoff: When markup changes, you fix the page object once—not 30 tests.

4) Keep DI/mocks thin and intentional

Spin up the smallest possible TestBed.

Provide only what you need.
Use jasmine.createSpyObj (or jest.fn()) for services.
Prefer HttpClientTestingModule over real HTTP.

beforeEach(async () => {
  mockApi = jasmine.createSpyObj('ApiService', ['getUser', 'updateUser']);

  await TestBed.configureTestingModule({
    imports: [ReactiveFormsModule, HttpClientTestingModule],
    declarations: [ProfileComponent],
    providers: [{ provide: ApiService, useValue: mockApi }],
  }).compileComponents();
});

Payoff: Fewer dependencies → fewer broken tests when modules shuffle.

5) Master Angular async: `fakeAsync`, `tick`, `waitForAsync`, and Observables

Pick one style per spec and use it consistently.

fakeAsync + tick(): great for timers, debounces, animations.
waitForAsync + fixture.whenStable(): great for async init with Promises.
marbles (optional): for complex RxJS flows.

it('debounces search input', fakeAsync(() => {
  page.typeSearch('mo');
  tick(300); // debounceTime(300)
  expect(mockApi.search).toHaveBeenCalledWith('mo');
}));

Payoff: Timing changes are easy to adjust; tests don’t flake.

6) Trigger change detection deliberately

Know when to call fixture.detectChanges():

Initial render after creating component.
After changing Inputs, updating form controls, or firing events.
After pushing values into Subjects in mocked services.

component.items = ['a', 'b'];
fixture.detectChanges();

Payoff: Predictable updates; fewer “why isn’t it rendering” moments.

7) Test Inputs & Outputs explicitly

When code changes, this is what matters most.

it('renders price with currency', () => {
  component.price = 1999; // cents
  component.currency = 'USD';
  fixture.detectChanges();

  expect(getEl(fixture, 'price').textContent).toContain('$19.99');
});

it('forwards edit click to parent', () => {
  const spy = jasmine.createSpy('edit');
  component.edit.subscribe(spy);

  getEl(fixture, 'edit-btn').click();
  expect(spy).toHaveBeenCalled();
});

8) Make HTTP tests change-proof with `HttpTestingController`

Don’t rely on real endpoints or interceptors.

it('loads profile on init', () => {
  fixture.detectChanges(); // ngOnInit
  const req = http.expectOne('/api/profile');
  req.flush({ name: 'Ada' });

  expect(component.form.value).toEqual({ name: 'Ada' });
});

Payoff: Endpoint contract changes are one-line fixes.

9) Time, dates, and randomness: freeze them

Refactors often change date formatting or default ranges. Freeze time & seeds.

Jasmine clock or Jest fake timers.
Override Date.now, use test doubles for date services.

beforeEach(() => spyOn(Date, 'now').and.returnValue(new Date('2025-01-01T00:00:00Z').getTime()));

Payoff: No flakiness from “today” logic.

10) Feature flags and environment

Test both sides of flags without rebuilding TestBed.

providers: [{ provide: FEATURE_FLAGS, useValue: { enableV2: true } }]

Tip: Wrap flags in a service or injection token so tests can flip them cheaply.

11) Keep assertions behavior-centric and minimal

Two or three strong assertions beat twenty brittle ones.

Assert what the user sees/does.
Avoid asserting CSS class lists unless they represent behavior (e.g., “disabled”).

12) When refactoring: protect with characterization tests

Before changing old, messy code:

Write a couple of tests that capture current (even if weird) behavior.
Refactor freely.
Keep or modify tests to represent the desired behavior.

Payoff: You never regress unintentionally.

13) Teardown & isolation

Angular tears down automatically, but explicitly enabling teardown helps catch leaks:

TestBed.configureTestingModule({ /* ... */ }).compileComponents();
// Angular 16+ does good defaults; if needed:
TestBed.resetTestingModule();

Use fresh fixtures per test; avoid shared mutable state.

14) Organize tests like the component’s contract

describe(Component):
  - rendering (inputs)
  - interaction (outputs, clicks, keyboard)
  - async flows (loading, success, error)
  - edge cases (null/undefined, permissions)

Payoff: When behavior changes, you know exactly where to update.

15) Tooling tips that spare you time

Angular Testing Library (ATL): ergonomic queries & user events.
Material Harnesses: stable access to Material components.
Coverage heatmaps: add small, targeted tests to flip the last red lines.
Watch mode + focused tests: fit/it.only while iterating (don’t commit them).

Mini Example: making a brittle test resilient

Before (brittle):

expect(fixture.nativeElement.querySelector('.btn.primary').textContent).toBe('Save');

After (resilient):

expect(getEl(fixture, 'save-btn').textContent).toContain('Save');

A short checklist when code changes

Did public inputs/outputs change? → Update tests to match the new contract.
Did markup change? → Update Page Object or test harness selectors (tests stay the same).
Did timing/debounce change? → Adjust tick()/fakeAsync() durations.
Did services/endpoints change? → Update HttpTestingController expectations only.
Did feature flags flip? → Provide a different flag value in the spec.

Templates you can copy

Data-test ID helper

export function click(fixture: ComponentFixture<any>, id: string) {
  const el = fixture.nativeElement.querySelector(`[data-testid="${id}"]`) as HTMLElement;
  el.click();
  fixture.detectChanges();
}

Spy service factory

function createApiSpy() {
  return jasmine.createSpyObj<ApiService>('ApiService', ['get', 'post', 'update']);
}

Mocking Observables cleanly

const subject = new BehaviorSubject<User | null>(null);
mockUserService.user$ = subject.asObservable();

subject.next({ id: '1', name: 'Ada' });
fixture.detectChanges();

Bottom line

Test behavior, not plumbing.
Stabilize selectors (test IDs / harnesses).
Abstract test mechanics (Page Objects, helpers).
Control time & async.
Keep DI/mocks small.

Do this and your Angular tests will survive refactors, design changes, and new requirements with minimal edits—exactly what you want when code inevitably evolves.

If you want, send me one of your brittle specs and the component—it’s quick to show how to “harden” it using the patterns above.

A Practical Strategy for Angular Testing

LS — Wed, 27 Aug 2025 13:17:16 +0000

Testing in Angular is both a discipline and an investment. Done well, it ensures confidence in rapid releases, prevents regressions, and makes code easier to maintain. But writing effective tests isn’t just about hitting 80% coverage—it’s about designing a strategy that balances unit tests, integration tests, and end-to-end (E2E) tests while keeping tests fast, reliable, and meaningful.

Testing Pyramid for Angular

A healthy Angular test suite should follow a pyramid-like structure:

Unit Tests (foundation, ~70%)
Fast, isolated, verifying a single component, service, or pipe.

Integration Tests (~20%)
Combine multiple Angular building blocks, e.g., a component with its service.

End-to-End Tests (~10%)
High-level UI flows using tools like Cypress or Playwright.

The goal is not “all tests everywhere,” but “right tests at the right level.”

Unit Testing Best Practices Services

Use HttpClientTestingModule with HttpTestingController to intercept API calls.

Mock dependencies with spies (jasmine.createSpyObj) instead of real implementations.

Test input → output: e.g., does saveBulkAction() call the correct URL with headers?

it('should PUT with headers and task body', () => {
service.saveBulkAction(payload, appId, true, false, 't-1', task).subscribe();

const req = httpMock.expectOne(r => r.urlWithParams.includes('application-bulk'));
expect(req.request.method).toBe('PUT');
expect(req.request.headers.get('Current-Entitlement')).toBe('ent-123');
expect(req.request.body.task).toEqual(task);
});

Components

Use Angular’s TestBed and shallow render where possible.

Stub child components with ng-mocks or NO_ERRORS_SCHEMA to avoid testing Angular itself.

Focus on inputs, outputs, and template bindings.

it('should emit value on button click', () => {
const button = fixture.debugElement.query(By.css('button'));
spyOn(component.submit, 'emit');
button.nativeElement.click();
expect(component.submit.emit).toHaveBeenCalled();
});

Pipes & Utilities

Keep these simple: one test per logical branch is enough.

Integration Testing

Sometimes you need to ensure multiple Angular parts work together:

Test a component with its real service but mock only HTTP calls.

Use TestBed.inject to bring in services and verify real Angular DI wiring.

Useful for catching template + service mismatches.

End-to-End Testing

Unit and integration tests cover correctness, but E2E ensures the user journey works:

Use Cypress or Playwright for realistic browser interactions.

Keep E2E tests short and focused on critical flows (login, form submit, checkout).

Run them in CI/CD nightly or pre-release; don’t block dev flow with slow suites.

Coverage vs. Confidence

Coverage is a metric, not a goal. Aim for meaningful coverage:

Cover decision branches (if/else, error handling).

Cover public APIs of services and components.

Don’t waste time on trivial Angular boilerplate (e.g., ngOnInit with no logic).

Practical Tips

LocalStorage / SessionStorage: Mock with spies in unit tests.

Async testing: Prefer fakeAsync with tick() over done() callbacks.

Error handling: Always test how services/components behave on error responses.

Test doubles: Use factories for mock data (avoid copy-pasting JSON blobs).

CI/CD: Fail fast—run unit tests on every push, E2E tests on merge to main.

Example Angular Testing Strategy (TL;DR)

Unit tests for every service, pipe, and component (70%).

Integration tests for components + services together (20%).

E2E tests for top workflows only (10%).

Focus on confidence, not coverage.

Keep tests fast, isolated, and readable.

how to write angular post

LS — Mon, 25 Aug 2025 22:28:23 +0000

Forem: LS

sql python note

flyway implementation

merge jsonschema

date conversion angular

How to use crypto library in Nodejs

mongodb shell script counting

NodeJs Mongo script query

Mongo db index creation

Mongo db index creation

How to handle changes in angular tests

Handling Change in Angular Tests (Without Losing Your Mind)

1) Adopt a “Public-Surface First” mindset

2) Stabilize DOM queries with test IDs (or Harnesses)

3) Use Page Objects for complex components

4) Keep DI/mocks thin and intentional

5) Master Angular async: fakeAsync, tick, waitForAsync, and Observables

6) Trigger change detection deliberately

7) Test Inputs & Outputs explicitly

8) Make HTTP tests change-proof with HttpTestingController

9) Time, dates, and randomness: freeze them

10) Feature flags and environment

11) Keep assertions behavior-centric and minimal

12) When refactoring: protect with characterization tests

13) Teardown & isolation

14) Organize tests like the component’s contract

15) Tooling tips that spare you time

Mini Example: making a brittle test resilient

A short checklist when code changes

Templates you can copy

Data-test ID helper

Spy service factory

Mocking Observables cleanly

Bottom line

A Practical Strategy for Angular Testing

how to write angular post

5) Master Angular async: `fakeAsync`, `tick`, `waitForAsync`, and Observables

8) Make HTTP tests change-proof with `HttpTestingController`