mirror/superpowers
1
0
Fork 0
mirror of https://github.com/obra/superpowers.git synced 2026-08-07 07:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(e2e): reject fenced and malformed scenario tables

Browse Source 
Filter backtick and tilde fenced examples before locating the E2E scenario-card table, so illustrative Markdown cannot satisfy the mechanical gate or shadow a later real table.\n\nParse only the documented leading/trailing-pipe row form and require a same-width delimiter row whose cells contain at least three hyphens with optional alignment colons. Missing or malformed delimiters now remain check failures instead of being silently treated as data.\n\nThe process-level harness records RED reproductions for both false-pass forms, fenced-only tables, and fenced examples before real tables, then verifies the focused GREEN behavior and prior contracts.
This commit is contained in:
Drew Ritter
2026-08-06 14:49:13 -07:00
parent f3a2fae88b
commit 2a2e0b41c2
2 changed files with 118 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/agentic-end-to-end-testing/scripts/check-cards-against-spec
+66 -10
View File
@@ -34,6 +34,35 @@ warn() { echo "warn: $1"; }
# design spec: markdown re-wrapping must not defeat the verbatim check.)
normalize() { tr -s '[:space:]' ' ' | sed -e 's/^ //' -e 's/ $//'; }
# Exclude fenced examples from structural matching. This intentionally models
# only backtick/tilde fences; it is not a general Markdown parser.
without_fenced_code() {
awk '
function fence_family(line, first, count) {
sub(/^[[:space:]]*/, "", line)
first = substr(line, 1, 1)
if (first != "`" && first != "~") return ""
count = 0
while (substr(line, count + 1, 1) == first) count++
return count >= 3 ? first : ""
}
{
marker = fence_family($0)
if (!in_fence && marker != "") {
in_fence = 1
family = marker
next
}
if (in_fence && marker == family) {
in_fence = 0
family = ""
next
}
if (!in_fence) print
}
' "$1"
}
# Text of the card's Expected section only (case-insensitive heading match,
# any ##+ level; section ends at the next heading or EOF).
expected_section() {
@@ -56,7 +85,7 @@ TABLE="$(awk '
}
insec && /^[[:space:]]*\|/ { intable = 1; print; next }
insec && intable { exit }
' "$SPEC")"
' < <(without_fenced_code "$SPEC"))"
if [ -z "$TABLE" ]; then
echo "no scenario table: $SPEC has no \"E2E scenario cards\" heading with a table under it" >&2
@@ -68,13 +97,24 @@ fi
US=$'\x1f'
CARD_COL=-1; FALS_COL=-1; ROWS=0
declare -a ROW_CARD ROW_FALS
HEADER_COLS=0; TABLE_VALID=1
lineno=0
while IFS= read -r line; do
lineno=$((lineno + 1))
esc="${line//\\|/$US}"
row="$(printf '%s' "$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
case "$row" in
\|*\|) ;;
*)
fail "row $lineno: canonical table rows require leading and trailing pipes"
TABLE_VALID=0
break
;;
esac
row="${row#|}"
row="${row%|}"
esc="${row//\\|/$US}"
IFS='|' read -r -a cells <<< "$esc"
# drop leading/trailing empty fields produced by the outer pipes
trimmed=()
for c in "${cells[@]}"; do
c="${c//$US/|}"
@@ -88,14 +128,28 @@ while IFS= read -r line; do
[ "$low" = "card" ] && CARD_COL=$i
[ "$low" = "falsification" ] && FALS_COL=$i
done
HEADER_COLS=${#trimmed[@]}
if [ "$CARD_COL" -lt 0 ] || [ "$FALS_COL" -lt 0 ]; then
fail "table header must name Card and Falsification columns"
TABLE_VALID=0
break
fi
continue
fi
# separator row: cells of dashes/colons only
joined="$(printf '%s' "${trimmed[*]}" | tr -d ' :-')"
[ -z "$joined" ] && continue
if [ "$CARD_COL" -lt 0 ] || [ "$FALS_COL" -lt 0 ]; then
fail "table header must name Card and Falsification columns"
break
if [ "$lineno" -eq 2 ]; then
delimiter_ok=1
[ "${#trimmed[@]}" -eq "$HEADER_COLS" ] || delimiter_ok=0
for c in "${trimmed[@]}"; do
if ! printf '%s\n' "$c" | grep -Eq '^:?-{3,}:?$'; then
delimiter_ok=0
fi
done
if [ "$delimiter_ok" -ne 1 ]; then
fail "row 2: malformed table delimiter"
TABLE_VALID=0
break
fi
continue
fi
card="${trimmed[$CARD_COL]:-}"
falsif="${trimmed[$FALS_COL]:-}"
@@ -107,7 +161,9 @@ while IFS= read -r line; do
ROW_CARD[$ROWS]="$card"; ROW_FALS[$ROWS]="$falsif"; ROWS=$((ROWS + 1))
done <<< "$TABLE"
[ "$ROWS" -ge 1 ] || fail "scenario table has no data rows"
if [ "$TABLE_VALID" -eq 1 ] && [ "$ROWS" -lt 1 ]; then
fail "scenario table has no data rows"
fi
# --- checks 2-4 per row -----------------------------------------------------
i=0
tests/agentic-e2e-checker/test-check-cards-against-spec.sh
+52
View File
@@ -202,6 +202,58 @@ assert_exit 0 "extra card -> exit 0" \
"$CHECKER" "$TEST_ROOT/t6/spec.md" "$TEST_ROOT/t6/cards"
assert_out_contains "extra-exploration" "warning names the extra card"
echo "scenario table requires a delimiter row"
make_spec "$TEST_ROOT/t10"; make_cards "$TEST_ROOT/t10/cards"
sed -i.bak '/^| --- | --- | --- |$/d' "$TEST_ROOT/t10/spec.md"
assert_exit 1 "header followed by data without delimiter -> exit 1" \
"$CHECKER" "$TEST_ROOT/t10/spec.md" "$TEST_ROOT/t10/cards"
echo "scenario table requires a valid delimiter row"
make_spec "$TEST_ROOT/t11"; make_cards "$TEST_ROOT/t11/cards"
sed -i.bak 's/^| --- | --- | --- |$/| -- | --- | --- |/' "$TEST_ROOT/t11/spec.md"
assert_exit 1 "delimiter cells require at least three hyphens -> exit 1" \
"$CHECKER" "$TEST_ROOT/t11/spec.md" "$TEST_ROOT/t11/cards"
echo "scenario table inside a fenced example does not count"
mkdir -p "$TEST_ROOT/t12/cards"
make_cards "$TEST_ROOT/t12/cards"
cat > "$TEST_ROOT/t12/spec.md" <<'EOF'
# Widget Design
## E2E scenario cards
```markdown
| Card | Covers | Falsification |
| --- | --- | --- |
| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. |
| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. |
```
EOF
assert_exit 2 "fenced-only table -> exit 2" \
"$CHECKER" "$TEST_ROOT/t12/spec.md" "$TEST_ROOT/t12/cards"
echo "fenced example before a real table is ignored"
mkdir -p "$TEST_ROOT/t13/cards"
make_cards "$TEST_ROOT/t13/cards"
cat > "$TEST_ROOT/t13/spec.md" <<'EOF'
# Widget Design
## E2E scenario cards
~~~markdown
| Card | Covers | Falsification |
| --- | --- | --- |
| fake-card | Example only | If the example is absent, the scenario FAILS. |
~~~
| Card | Covers | Falsification |
| --- | --- | --- |
| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. |
| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. |
EOF
assert_exit 0 "real table after fenced example -> exit 0" \
"$CHECKER" "$TEST_ROOT/t13/spec.md" "$TEST_ROOT/t13/cards"
echo "no scenario table"
mkdir -p "$TEST_ROOT/t7/cards"
printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md"