diff --git a/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec b/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec index dc6281da..149cdc4b 100755 --- a/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec +++ b/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec @@ -73,7 +73,7 @@ expected_section() { if (insec) exit } insec { print } - ' "$1" + ' } # --- extract the first table under the (case-insensitive) heading ---------- @@ -90,6 +90,7 @@ TABLE="$(awk ' if [ -z "$TABLE" ]; then echo "no scenario table: $SPEC has no \"E2E scenario cards\" heading with a table under it" >&2 echo "(heading must be exactly \"E2E scenario cards\" — no numbering or extra words)" >&2 + echo "(scenario table rows must use leading and trailing outer pipes)" >&2 exit 2 fi @@ -121,7 +122,6 @@ while IFS= read -r line; do c="$(printf '%s' "$c" | normalize)" trimmed+=("$c") done - # cells[0] is empty (before first |); last may be empty too if [ "$lineno" -eq 1 ]; then for i in "${!trimmed[@]}"; do low="$(printf '%s' "${trimmed[$i]}" | tr '[:upper:]' '[:lower:]')" @@ -153,11 +153,20 @@ while IFS= read -r line; do fi card="${trimmed[$CARD_COL]:-}" falsif="${trimmed[$FALS_COL]:-}" - card="${card//\`/}" # tolerate `card-name` backticks in the cell if [ -z "$card" ] || [ -z "$falsif" ]; then fail "row $lineno: empty Card or Falsification cell" continue fi + case "$card" in + \`*\`) + card="${card#\`}" + card="${card%\`}" + ;; + esac + if ! [[ "$card" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then + fail "row $lineno: invalid Card value: $card" + continue + fi ROW_CARD[$ROWS]="$card"; ROW_FALS[$ROWS]="$falsif"; ROWS=$((ROWS + 1)) done <<< "$TABLE" @@ -174,15 +183,16 @@ while [ "$i" -lt "$ROWS" ]; do fail "missing card file: $f" i=$((i + 1)); continue fi - hay="$(expected_section "$f" | normalize)" + card_text="$(without_fenced_code "$f")" + hay="$(printf '%s\n' "$card_text" | expected_section | normalize)" case "$hay" in *"$falsif"*) : ;; *) fail "$f: falsification line not present verbatim in the ## Expected section. expected (normalized): $falsif" ;; esac - grep -q '\*\*What this covers\*\*' "$f" || fail "$f: missing **What this covers**" + grep -q '\*\*What this covers\*\*' <<< "$card_text" || fail "$f: missing **What this covers**" for sec in Pre-state Steps Expected Cleanup; do - grep -Eiq "^#{2,}[[:space:]]*${sec}[[:space:]]*$" "$f" || fail "$f: missing ## ${sec} section" + grep -Eiq "^#{2,}[[:space:]]*${sec}[[:space:]]*$" <<< "$card_text" || fail "$f: missing ## ${sec} section" done i=$((i + 1)) done diff --git a/tests/agentic-e2e-checker/test-check-cards-against-spec.sh b/tests/agentic-e2e-checker/test-check-cards-against-spec.sh index 4dbe103a..d672005c 100755 --- a/tests/agentic-e2e-checker/test-check-cards-against-spec.sh +++ b/tests/agentic-e2e-checker/test-check-cards-against-spec.sh @@ -254,6 +254,79 @@ EOF assert_exit 0 "real table after fenced example -> exit 0" \ "$CHECKER" "$TEST_ROOT/t13/spec.md" "$TEST_ROOT/t13/cards" +echo "card structure inside a fenced example does not count" +make_spec "$TEST_ROOT/t14"; make_cards "$TEST_ROOT/t14/cards" +cat > "$TEST_ROOT/t14/cards/widget-show-table.md" <<'EOF' +# widget-show-table: illustrative card only + +~~~markdown +**What this covers**: the rendered table. + +## Pre-state +A built widget binary. + +## Steps +1. Run `widget show`. + +## Expected +If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. + +## Cleanup +Nothing to clean. +~~~ +EOF +assert_exit 1 "required card content found only inside a fence -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t14/spec.md" "$TEST_ROOT/t14/cards" + +echo "unrelated fenced examples remain valid card content" +make_spec "$TEST_ROOT/t15"; make_cards "$TEST_ROOT/t15/cards" +cat >> "$TEST_ROOT/t15/cards/widget-show-table.md" <<'EOF' + +```console +$ widget show +TOTAL 20.85 +``` +EOF +assert_exit 0 "valid card with unrelated fenced example -> exit 0" \ + "$CHECKER" "$TEST_ROOT/t15/spec.md" "$TEST_ROOT/t15/cards" + +echo "card names cannot traverse outside the cards directory" +make_spec "$TEST_ROOT/t16"; make_cards "$TEST_ROOT/t16/cards" +sed -i.bak 's/| widget-show-table |/| ..\/outside |/' "$TEST_ROOT/t16/spec.md" +mv "$TEST_ROOT/t16/cards/widget-show-table.md" "$TEST_ROOT/t16/outside.md" +assert_exit 1 "parent-traversing Card value -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t16/spec.md" "$TEST_ROOT/t16/cards" +assert_out_contains "invalid Card value" "traversal failure names the invalid value" + +echo "card names use lowercase kebab case" +make_spec "$TEST_ROOT/t17"; make_cards "$TEST_ROOT/t17/cards" +sed -i.bak 's/| widget-show-table |/| Upper-Case |/' "$TEST_ROOT/t17/spec.md" +mv "$TEST_ROOT/t17/cards/widget-show-table.md" "$TEST_ROOT/t17/cards/Upper-Case.md" +assert_exit 1 "uppercase Card value -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t17/spec.md" "$TEST_ROOT/t17/cards" +assert_out_contains "invalid Card value" "uppercase failure names the invalid value" + +make_spec "$TEST_ROOT/t18"; make_cards "$TEST_ROOT/t18/cards" +sed -i.bak 's/| widget-show-table |/| two--segments |/' "$TEST_ROOT/t18/spec.md" +mv "$TEST_ROOT/t18/cards/widget-show-table.md" "$TEST_ROOT/t18/cards/two--segments.md" +assert_exit 1 "empty kebab segment -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t18/spec.md" "$TEST_ROOT/t18/cards" +assert_out_contains "invalid Card value" "empty-segment failure names the invalid value" + +echo "one enclosing backtick pair remains supported" +make_spec "$TEST_ROOT/t19"; make_cards "$TEST_ROOT/t19/cards" +sed -i.bak 's/| widget-show-table |/| `widget-show-table` |/' "$TEST_ROOT/t19/spec.md" +assert_exit 0 "backticked kebab-case Card value -> exit 0" \ + "$CHECKER" "$TEST_ROOT/t19/spec.md" "$TEST_ROOT/t19/cards" + +echo "pipe-less tables report the canonical outer-pipe contract" +make_spec "$TEST_ROOT/t20"; make_cards "$TEST_ROOT/t20/cards" +sed -E -i.bak 's/^\| (.*) \|$/\1/' "$TEST_ROOT/t20/spec.md" +assert_exit 2 "pipe-less table remains unsupported -> exit 2" \ + "$CHECKER" "$TEST_ROOT/t20/spec.md" "$TEST_ROOT/t20/cards" +assert_out_contains "scenario table rows must use leading and trailing outer pipes" \ + "diagnostic names the canonical outer-pipe contract" + echo "no scenario table" mkdir -p "$TEST_ROOT/t7/cards" printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md"