atago

Summary #

83 suites · 683 scenarios

Contents #

atago self-hosting / cross-platform no-shell argv tokenization (#154) #

Source: test/e2e/atago/argv_quotes.atago.yaml

Scenario: a single-quoted JSON argument survives tokenization #

When #

${atago} run '{"k":"v"}'

Then #

  • exit code is 3
  • stderr contains {\"k\":\"v\"}

Scenario: a single-quoted argument with a space stays one argument #

When #

${atago} run 'no such file.yaml'

Then #

  • exit code is 3
  • stderr contains no such file.yaml

Scenario: a block-scalar command splits on newlines like spaces #

When #

${atago} run
no-such-file.yaml

Then #

  • exit code is 3
  • stderr contains no-such-file.yaml

Scenario: a folded-scalar command drops its trailing newline #

When #

${atago} run no-such-file.yaml

Then #

  • exit code is 3
  • stderr contains no-such-file.yaml

atago self-hosting / artifacts-dir failure payloads #

Source: test/e2e/atago/artifacts.atago.yaml

Scenario: a failing stdout equals writes expected and actual sidecars #

Given #

  • Fixture file fail.atago.yaml is created.

Inputs #

Fixture fail.atago.yaml:

version: "1"
suite: {name: fail}
scenarios:
  - name: stdout drifts
    steps:
      - run: {shell: true, command: "printf 'actual-value\\n'"}
      - assert: {stdout: {equals: "expected-value"}}

When #

${atago} run --artifacts-dir arts fail.atago.yaml
cat arts/*/*/*.actual.txt arts/*/*/*.expected.txt

Then #

  • after ${atago} run --artifacts-dir arts fail.atago.yaml:
    • exit code is 1
  • after cat arts/*/*/*.actual.txt arts/*/*/*.expected.txt:
    • exit code is 0
    • stdout contains actual-value, expected-value

Scenario: a passing scenario writes no failure payload #

Given #

  • Fixture file pass.atago.yaml is created.

Inputs #

Fixture pass.atago.yaml:

version: "1"
suite: {name: pass}
scenarios:
  - name: all good
    steps:
      - run: {shell: true, command: "printf 'ok\\n'"}
      - assert: {stdout: {equals: "ok"}}

When #

${atago} run --artifacts-dir arts2 pass.atago.yaml
find arts2 -name '*.actual.txt' 2>/dev/null | wc -l

Then #

  • after ${atago} run --artifacts-dir arts2 pass.atago.yaml:
    • exit code is 0
  • after find arts2 -name '*.actual.txt' 2>/dev/null | wc -l:
    • stdout matches /^\s*0\s*$/

Scenario: the artifacts directory is created when it does not exist #

Given #

  • Fixture file nested.atago.yaml is created.

Inputs #

Fixture nested.atago.yaml:

version: "1"
suite: {name: nested}
scenarios:
  - name: fails
    steps:
      - run: {shell: true, command: "printf 'x\\n'"}
      - assert: {stdout: {equals: "y"}}

When #

${atago} run --artifacts-dir deep/made/arts nested.atago.yaml

Then #

  • exit code is 1
  • dir deep/made/arts exists

Scenario: each repeat iteration keeps its own failure payloads #

Given #

  • Fixture file repeated.atago.yaml is created.

Inputs #

Fixture repeated.atago.yaml:

version: "1"
suite: {name: repeated}
scenarios:
  - name: prints its own workdir
    steps:
      # $${workdir} escapes the OUTER expansion, so the inner run
      # prints its own per-iteration workdir instead of a constant.
      - run: {shell: true, command: "echo $${workdir}"}
      - assert: {stdout: {equals: "never-this"}}

When #

${atago} run --repeat 3 --artifacts-dir reparts repeated.atago.yaml
find reparts -type d -name 'attempt-*' | sort | sed 's#.*/##'
find reparts -name '*.actual.txt' | wc -l | tr -d ' '
cat $(find reparts -name '*.actual.txt') | sort -u | wc -l | tr -d ' '

Then #

  • after ${atago} run --repeat 3 --artifacts-dir reparts repeated.atago.yaml:
    • exit code is 1
  • after find reparts -type d -name 'attempt-*' | sort | sed 's#.*/##':
    • exit code is 0
    • stdout equals an exact value
  • after find reparts -name '*.actual.txt' | wc -l | tr -d ' ':
    • stdout equals an exact value
  • after cat $(find reparts -name '*.actual.txt') | sort -u | wc -l | tr -d ' ':
    • stdout equals an exact value

Expected output #

expected stdout:

attempt-2
attempt-3

Scenario: a file-content mismatch also writes a payload #

Given #

  • Fixture file filefail.atago.yaml is created.

Inputs #

Fixture filefail.atago.yaml:

version: "1"
suite: {name: filefail}
scenarios:
  - name: file drifts
    steps:
      - fixture: {file: out.txt, content: "written-bytes"}
      - assert: {file: {path: out.txt, equals: "wanted-bytes"}}

When #

${atago} run --artifacts-dir farts filefail.atago.yaml
find farts -type f | wc -l

Then #

  • after ${atago} run --artifacts-dir farts filefail.atago.yaml:
    • exit code is 1
  • after find farts -type f | wc -l:
    • stdout does not match /^\s*0\s*$/

Scenario: a teardown failure keeps the steps failure payloads #

Given #

  • Fixture file bothfail.atago.yaml is created.

Inputs #

Fixture bothfail.atago.yaml:

version: "1"
suite: {name: bothfail}
scenarios:
  - name: both phases fail at index 1
    steps:
      - run: {shell: true, command: "printf 'STEPS-ACTUAL\\n'"}
      - assert: {stdout: {equals: "STEPS-EXPECTED"}}
    teardown:
      - run: {shell: true, command: "printf 'TEARDOWN-ACTUAL\\n'"}
      - assert: {stdout: {equals: "TEARDOWN-EXPECTED"}}

When #

${atago} run --artifacts-dir tarts bothfail.atago.yaml
cat tarts/*/*/step-01-stdout.actual.txt
cat tarts/*/*/teardown/step-01-stdout.actual.txt
find tarts -name '*.actual.txt' | wc -l | tr -d ' '

Then #

  • after ${atago} run --artifacts-dir tarts bothfail.atago.yaml:
    • exit code is 1
  • after cat tarts/*/*/step-01-stdout.actual.txt:
    • exit code is 0
    • stdout equals an exact value
  • after cat tarts/*/*/teardown/step-01-stdout.actual.txt:
    • exit code is 0
    • stdout equals an exact value
  • after find tarts -name '*.actual.txt' | wc -l | tr -d ' ':
    • stdout equals an exact value

atago self-hosting / variable expansion in assertion matcher values #

Source: test/e2e/atago/assert_expand.atago.yaml

Scenario: stdout.equals expands ${workdir} #

When #

printf "%s\n" "${workdir}/out.txt"

Then #

  • stdout equals an exact value

Scenario: stdout.contains and not_contains expand a stored variable #

When #

printf "%s" hello-123
# capture ${token} from stdout
printf "%s\n" "hello-123 world"

Then #

  • after printf "%s\n" "hello-123 world":
    • stdout contains ${token}
    • stdout does not contain ${token}-absent

Scenario: file.contains expands ${workdir} #

When #

printf "%s\n" "${workdir}/marker" > note.txt

Then #

  • file note.txt contains ${workdir}/marker

Scenario: dir.path expands a stored variable #

When #

mkdir -p site && touch site/a.html && echo site
# capture ${outdir} from stdout

Then #

  • dir ${outdir} contains a.html

Scenario: changes entries expand a stored variable #

When #

echo out
# capture ${base} from stdout
echo w > ${base}2.txt

Then #

  • after echo w > ${base}2.txt:
    • the step changed exactly created ${base}2.txt, modified nothing, deleted nothing

Scenario: screen matcher expands a stored variable #

skipped on Windows

When #

echo needle
# capture ${pat} from stdout
# interactive (pty): echo needle

Then #

  • after interactive (pty): echo needle:
    • rendered screen contains ${pat}

atago self-hosting / browser (cdp) runner #

Source: test/e2e/atago/cdp.atago.yaml

Scenario: a cdp step with no actions fails validation (exit 2) #

Given #

  • Fixture file badcdp.atago.yaml is created.

Inputs #

Fixture badcdp.atago.yaml:

version: "1"
suite:
  name: browser
runners:
  web:
    type: browser
scenarios:
  - name: empty actions
    steps:
      - cdp:
          runner: web
          actions: []

When #

${atago} run badcdp.atago.yaml

Then #

  • exit code is 2
  • stderr contains at least one action

Scenario: a cdp step naming an undeclared runner fails validation (exit 2) #

Given #

  • Fixture file norunner.atago.yaml is created.

Inputs #

Fixture norunner.atago.yaml:

version: "1"
suite:
  name: browser
scenarios:
  - name: missing runner
    steps:
      - cdp:
          runner: missing
          actions:
            - navigate: http://example.com

When #

${atago} run norunner.atago.yaml

Then #

  • exit code is 2
  • stderr contains is not declared

Scenario: a screenshot action without a path fails validation (exit 2) #

Given #

  • Fixture file noshotpath.atago.yaml is created.

Inputs #

Fixture noshotpath.atago.yaml:

version: "1"
suite:
  name: browser
runners:
  web:
    type: browser
scenarios:
  - name: screenshot missing path
    steps:
      - cdp:
          runner: web
          actions:
            - navigate: http://example.com
            - screenshot:
                selector: "#x"

When #

${atago} run noshotpath.atago.yaml

Then #

  • exit code is 2
  • stderr contains screenshot requires a path

Scenario: explain lists the extended cdp actions #

Given #

  • Fixture file actions.atago.yaml is created.

Inputs #

Fixture actions.atago.yaml:

version: "1"
suite:
  name: browser
runners:
  web:
    type: browser
scenarios:
  - name: a black-box ui flow
    steps:
      - cdp:
          runner: web
          actions:
            - navigate: http://example.com
            - wait_hidden: "#spinner"
            - press: { selector: "#in", key: Enter }
            - select: { selector: "#s", value: b }
            - check: "#agree"
            - screenshot: { path: shot.png }
            - title: true
            - attribute: { selector: "#lnk", name: href }

When #

${atago} explain actions.atago.yaml

Then #

  • exit code is 0
  • stdout contains wait_hidden, screenshot shot.png, attribute href

Scenario: a browser-only field on a non-browser runner fails validation (exit 2) #

Given #

  • Fixture file crosstype.atago.yaml is created.

Inputs #

Fixture crosstype.atago.yaml:

version: "1"
suite:
  name: browser
runners:
  api:
    type: http
    base_url: http://x
    headless: false
scenarios:
  - name: misuse
    steps:
      - run: { command: echo hi }

When #

${atago} run crosstype.atago.yaml

Then #

  • exit code is 2
  • stderr contains cannot be set on a http runner

Scenario: manifest surfaces the browser-runner configuration #

Given #

  • Fixture file cfg.atago.yaml is created.

Inputs #

Fixture cfg.atago.yaml:

version: "1"
suite:
  name: browser
runners:
  web:
    type: browser
    headless: false
    exec_path: /usr/bin/chromium
    browser_args: ["disable-gpu", "window-size=1280,720"]
scenarios:
  - name: ui
    steps:
      - cdp:
          runner: web
          actions:
            - navigate: http://example.com
            - title: true

When #

${atago} manifest cfg.atago.yaml

Then #

  • exit code is 0
  • stdout at $.specs[0].runners[0].headless equals false
  • stdout at $.specs[0].runners[0].exec_path equals /usr/bin/chromium

Scenario: an upload action without a file fails validation (exit 2) #

Given #

  • Fixture file badupload.atago.yaml is created.

Inputs #

Fixture badupload.atago.yaml:

version: "1"
suite:
  name: browser
runners:
  web:
    type: browser
scenarios:
  - name: upload missing file
    steps:
      - cdp:
          runner: web
          actions:
            - upload: { selector: "#f" }

When #

${atago} run badupload.atago.yaml

Then #

  • exit code is 2
  • stderr contains upload requires selector and file

Scenario: a download action without a click selector fails validation (exit 2) #

Given #

  • Fixture file baddownload.atago.yaml is created.

Inputs #

Fixture baddownload.atago.yaml:

version: "1"
suite:
  name: browser
runners:
  web:
    type: browser
scenarios:
  - name: download missing click
    steps:
      - cdp:
          runner: web
          actions:
            - download: { dir: out }

When #

${atago} run baddownload.atago.yaml

Then #

  • exit code is 2
  • stderr contains download requires a click selector

Scenario: a navigate to a denied host is a policy violation (exit 6) #

Given #

  • Fixture file policy.atago.yaml is created.

Inputs #

Fixture policy.atago.yaml:

version: "1"
suite:
  name: browser
permissions:
  network:
    allow: ["allowed.example"]
runners:
  web:
    type: browser
scenarios:
  - name: navigates to a host the policy does not name
    steps:
      - cdp:
          runner: web
          actions:
            - navigate: https://denied.example/

When #

${atago} run policy.atago.yaml

Then #

  • exit code is 6
  • stdout contains network policy denies host "denied.example", holds `navigate:` to the policy

atago self-hosting / changes (workdir delta assertions) #

Source: test/e2e/atago/changes.atago.yaml

Scenario: a generator touches exactly the files it should (POSIX) #

skipped on Windows

Given #

  • Fixture file config.yaml is created.
  • Fixture file stale.html is created.

Inputs #

Fixture config.yaml:

theme: light

Fixture stale.html:

old

When #

printf 'theme: dark\n' > config.yaml && rm stale.html && mkdir -p site/assets && printf '<html></html>' > site/index.html && printf 'body{}' > site/assets/app.css

Then #

  • exit code is 0
  • the step changed exactly created site/index.html, site/assets/*.css, modified config.yaml, deleted stale.html

Scenario: an unexpected creation breaks the exact contract (POSIX) #

skipped on Windows

Given #

  • Fixture file check.atago.yaml is created.

Inputs #

Fixture check.atago.yaml:

version: "1"
suite:
  name: unexpected
scenarios:
  - name: extra file
    steps:
      - run:
          shell: true
          command: 'echo a > a.txt && echo b > b.txt'
      - assert:
          changes:
            created:
              - a.txt

When #

${atago} run check.atago.yaml

Then #

  • exit code is 1
  • stdout contains unexpected created file

Scenario: stdout_to counts as created, and modified nothing holds (portable) #

Given #

  • Fixture file input.txt is created.

Inputs #

Fixture input.txt:

seed

When #

echo produced

Then #

  • exit code is 0
  • the step changed exactly created result.txt, modified nothing, deleted nothing

Generated artifacts #

  • result.txt

Scenario: the delta over a retried step reflects only the converged attempt (POSIX) #

skipped on Windows

Given #

  • The step is retried up to 3 times every 10ms until exit code is 0.

When #

touch a; [ -f b ] && touch c; touch b; [ -f c ]

Then #

  • exit code is 0
  • the step changed exactly created c, modified nothing, deleted nothing

Scenario: deleting and recreating a byte-identical file appears in no list (POSIX) #

skipped on Windows

When #

printf hello > f.txt
rm f.txt && printf hello > f.txt

Then #

  • after rm f.txt && printf hello > f.txt:
    • exit code is 0
    • the step changed exactly created nothing, modified nothing, deleted nothing

Scenario: deleting and recreating with different content is modified only (POSIX) #

skipped on Windows

Given #

  • Fixture file f.txt is created.

Inputs #

Fixture f.txt:

hello

When #

rm f.txt && printf world > f.txt

Then #

  • exit code is 0
  • the step changed exactly created nothing, modified f.txt, deleted nothing

Scenario: stdout_to overwrites a fixture (modified) while stderr_to creates an empty file (POSIX) #

skipped on Windows

Given #

  • Fixture file existing.txt is created.

Inputs #

Fixture existing.txt:

old

When #

printf newcontent

Then #

  • exit code is 0
  • the step changed exactly created err.txt, modified existing.txt, deleted nothing

Generated artifacts #

  • existing.txt
  • err.txt

Scenario: a pty step feeds the delta scan just like a run step (POSIX) #

skipped on Windows

When #

# interactive (pty): sh -c "touch from-pty; echo done"

Then #

  • the step changed exactly created from-pty

Scenario: a doublestar glob pins an arbitrary-depth generated tree exactly (POSIX) #

skipped on Windows

When #

mkdir -p out/a/b && printf '1' > out/top.txt && printf '2' > out/a/mid.txt && printf '3' > out/a/b/leaf.txt

Then #

  • exit code is 0
  • the step changed exactly created out/**, modified nothing, deleted nothing

Scenario: a stray file outside the doublestar prefix breaks the exact contract (POSIX) #

skipped on Windows

Given #

  • Fixture file check.atago.yaml is created.

Inputs #

Fixture check.atago.yaml:

version: "1"
suite:
  name: stray-outside-doublestar
scenarios:
  - name: extra file beside the tree
    steps:
      - run:
          shell: true
          command: 'mkdir -p out/a && printf x > out/a/leaf.txt && printf y > stray.txt'
      - assert:
          changes:
            created:
              - "out/**"

When #

${atago} run check.atago.yaml

Then #

  • exit code is 1
  • stdout contains unexpected created file

Scenario: a doublestar glob matches a nested redirect target (portable) #

When #

echo produced

Then #

  • exit code is 0
  • the step changed exactly created out/**, modified nothing, deleted nothing

Generated artifacts #

  • out/deep/result.txt

Scenario: a doublestar prefix covers both redirect streams (portable) #

When #

echo out

Then #

  • exit code is 0
  • the step changed exactly created logs/**, modified nothing, deleted nothing

Generated artifacts #

  • logs/out.txt
  • logs/err.txt

skipped on Windows

Given #

  • Fixture file real.txt is created.

Inputs #

Fixture real.txt:

payload

When #

ln -s real.txt link.txt

Then #

  • exit code is 0
  • the step changed exactly created link.txt, modified nothing, deleted nothing

skipped on Windows

Given #

  • Fixture file check.atago.yaml is created.

Inputs #

Fixture check.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: claims to create nothing
    steps:
      - run:
          shell: true
          command: ln -s /etc/passwd sneaky
      - assert:
          exit_code: 0
          changes:
            created: []

When #

${atago} run check.atago.yaml

Then #

  • exit code is 1
  • stdout contains unexpected created file

skipped on Windows

Given #

  • Fixture file v1.txt is created.
  • Fixture file v2.txt is created.
  • Fixture file current is created.

Inputs #

Fixture v1.txt:

one

Fixture v2.txt:

two

When #

ln -sfn v2.txt current

Then #

  • exit code is 0
  • the step changed exactly created nothing, modified current, deleted nothing

skipped on Windows

Given #

  • Fixture file real.txt is created.
  • Fixture file link.txt is created.

Inputs #

Fixture real.txt:

before

When #

printf 'after\n' > link.txt

Then #

  • exit code is 0
  • the step changed exactly created nothing, modified real.txt, deleted nothing

skipped on Windows

Given #

  • Fixture file real.txt is created.
  • Fixture file link.txt is created.

Inputs #

Fixture real.txt:

payload

When #

rm link.txt

Then #

  • exit code is 0
  • the step changed exactly created nothing, modified nothing, deleted link.txt

skipped on Windows

When #

ln -s no/such/target dangling

Then #

  • exit code is 0
  • the step changed exactly created dangling, modified nothing, deleted nothing

Scenario: a changes entry naming a directory says what the path is #

Given #

  • Fixture file direntry.atago.yaml is created.

Inputs #

Fixture direntry.atago.yaml:

version: "1"
suite: {name: direntry}
scenarios:
  - name: the generator made a directory
    steps:
      - run: {shell: true, command: "mkdir out"}
      - assert:
          changes:
            created: [out]

When #

${atago} run direntry.atago.yaml

Then #

  • exit code is 1
  • stdout contains "out" exists as a directory, tracks only regular files and symlinks, assert it with dir:

Scenario: an ignored path does not break an exhaustive delta #

Given #

  • Fixture file strict.atago.yaml is created.

Inputs #

Fixture strict.atago.yaml:

version: "1"
suite: {name: strict}
scenarios:
  - name: an unignored surprise still fails
    steps:
      - run: {shell: true, command: "mkdir -p cache && printf 'x' > cache/state.bin && printf 'y' > surprise.txt"}
      - assert:
          changes:
            ignore: ["cache/**"]
            created: []

When #

mkdir -p cache && printf 'x' > cache/state.bin && printf 'y' > out.txt
${atago} run strict.atago.yaml

Then #

  • after mkdir -p cache && printf 'x' > cache/state.bin && printf 'y' > out.txt:
    • exit code is 0
    • the step changed exactly created out.txt, modified nothing, deleted nothing, ignoring cache/**
  • after ${atago} run strict.atago.yaml:
    • exit code is 1
    • stdout contains unexpected created file "surprise.txt", created [surprise.txt], modified [], deleted [], ignored [cache/state.bin]

Scenario: an ignored path cannot satisfy an entry that names it #

Given #

  • Fixture file contradiction.atago.yaml is created.

Inputs #

Fixture contradiction.atago.yaml:

version: "1"
suite: {name: contradiction}
scenarios:
  - name: ignored and expected at once
    steps:
      - run: {shell: true, command: "printf 'y' > out.txt"}
      - assert:
          changes:
            ignore: [out.txt]
            created: [out.txt]

When #

${atago} run contradiction.atago.yaml

Then #

  • exit code is 1
  • stdout contains created entry "out.txt" matched no file the step created

Scenario: a permission-only change is a modification #

skipped on Windows

Given #

  • Fixture file cfg.yaml is created.

Inputs #

Fixture cfg.yaml:

k: v

When #

chmod 777 cfg.yaml

Then #

  • exit code is 0
  • the step changed exactly created nothing, modified cfg.yaml, deleted nothing
  • file cfg.yaml is executable

Scenario: a planted fifo is a creation and is never opened #

skipped on Windows

Given #

  • Fixture file blind.atago.yaml is created.

Inputs #

Fixture blind.atago.yaml:

version: "1"
suite: {name: blind}
scenarios:
  - name: an undeclared fifo is not invisible
    steps:
      - run: {shell: true, command: "mkfifo surprise"}
      - assert:
          changes:
            created: []

When #

mkfifo pipe
${atago} run blind.atago.yaml

Then #

  • after mkfifo pipe:
    • exit code is 0
    • the step changed exactly created pipe, modified nothing, deleted nothing
  • after ${atago} run blind.atago.yaml:
    • exit code is 1
    • stdout contains unexpected created file "surprise"

atago self-hosting / CLI scenario selection #

Source: test/e2e/atago/cli_selection.atago.yaml

Scenario: filter selects by a name substring #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json --filter alpha inner.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 1
  • stdout contains "alpha", does not contain "beta", "gamma"

Scenario: filter is OR across a comma-separated list #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json --filter alpha,beta inner.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 2
  • stdout contains "alpha", "beta", does not contain "gamma"

Scenario: tag selects scenarios carrying the tag #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json --tag fast inner.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 2
  • stdout contains "alpha", "gamma", does not contain "beta"

Scenario: a repeated tag flag is OR #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json --tag fast --tag slow inner.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 3

Scenario: skip-tag removes scenarios carrying the tag #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json --skip-tag slow inner.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 1
  • stdout contains "alpha", does not contain "beta", "gamma"

Scenario: tag and skip-tag compose as selected minus skipped #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json --tag fast --skip-tag slow inner.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 1
  • stdout contains "alpha", does not contain "gamma"

Scenario: an empty filter selection fails under –ci with a substring hint #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json --filter no_such_name inner.atago.yaml

Then #

  • exit code is 3
  • stdout at $.suites[0].scenarios has length 0
  • stderr contains no scenarios matched, --filter "no_such_name", case-sensitive substring

Scenario: an empty tag selection fails under –ci and names the exact-tag rule #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json --tag no_such_tag inner.atago.yaml

Then #

  • exit code is 3
  • stderr contains no scenarios matched, --tag "no_such_tag", match tags exactly, atago list, does not contain case-sensitive substring

Scenario: without –ci an empty selection only warns and still exits zero #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite: {name: inner}
scenarios:
  - name: alpha
    tags: [fast]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: beta
    tags: [slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]
  - name: gamma
    tags: [fast, slow]
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --report json --filter no_such_name inner.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 0
  • stderr contains warning: no scenarios matched

atago self-hosting / completion #

Source: test/e2e/atago/completion.atago.yaml

Scenario: bash completion emits a recognizable script #

When #

${atago} completion bash

Then #

  • exit code is 0
  • stdout contains complete -F _atago atago

Scenario: zsh completion emits a compdef script #

When #

${atago} completion zsh

Then #

  • exit code is 0
  • stdout contains #compdef atago

Scenario: fish completion emits complete directives #

When #

${atago} completion fish

Then #

  • exit code is 0
  • stdout contains complete -c atago

Scenario: powershell completion registers an argument completer #

When #

${atago} completion powershell

Then #

  • exit code is 0
  • stdout contains Register-ArgumentCompleter

Scenario: unknown shell is a configuration error #

When #

${atago} completion tcsh

Then #

  • exit code is 3
  • stderr contains unknown shell

atago self-hosting / occurrence counts and byte sizes #

Source: test/e2e/atago/count_size.atago.yaml

Scenario: an exact count distinguishes once from twice #

When #

printf 'error: boom\nfine\n'

Then #

  • stdout contains error: exactly 1 time

Scenario: a count of zero is how a spec says never #

When #

echo ok

Then #

  • stdout contains panic never

Scenario: min and max bound a range a single number cannot #

When #

printf '1,a\n2,b\n3,c\n'

Then #

  • stdout matches /(?m)^[0-9]+,/ between 3 and 10 times

Scenario: a regexp counts its own non-overlapping matches #

When #

echo aaa

Then #

  • stdout matches /aa/ exactly 1 time

Scenario: a count applies to a file’s content too #

Given #

  • Fixture file log.txt is created.

Inputs #

Fixture log.txt:

hit
miss
hit

Then #

  • file log.txt contains hit exactly 2 times

Scenario: a duplicated line fails the inner spec and names where it landed #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: the error is printed twice
    steps:
      - run:
          shell: true
          command: "printf 'boom\\nfine\\nboom\\n'"
      - assert:
          stdout:
            contains: boom
            count: 1

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains occurs 2 times, line 1, line 3

Scenario: size zero pins a file that was created but left empty #

When #

printf '' > empty.txt

Then #

  • file empty.txt is empty

Scenario: size bounds compose with a content matcher #

Given #

  • Fixture file five.txt is created.

Inputs #

Fixture five.txt:

abcde

Then #

  • file five.txt contains abc and is exactly 5 bytes
  • file five.txt is between 1 and 4096 bytes

Scenario: a size miss by one byte names the usual cause #

Given #

  • Fixture file inner_size.atago.yaml is created.

Inputs #

Fixture inner_size.atago.yaml:

version: "1"
suite:
  name: inner size
scenarios:
  - name: the file carries a trailing newline
    steps:
      - fixture:
          file: nl.txt
          content: "abc\n"
      - assert:
          file:
            path: nl.txt
            size: 3

When #

${atago} run inner_size.atago.yaml

Then #

  • exit code is 1
  • stdout contains trailing newline

Scenario: a count without a countable matcher is a load error #

Given #

  • Fixture file bad_count.atago.yaml is created.

Inputs #

Fixture bad_count.atago.yaml:

version: "1"
suite:
  name: bad count
scenarios:
  - name: nothing to count
    steps:
      - run:
          command: echo hi
      - assert:
          stdout:
            equals: hi
            count: 1

When #

${atago} run bad_count.atago.yaml

Then #

  • exit code is 2
  • stderr contains need a contains or matches matcher to count

Scenario: an ambiguous count over two countable matchers is a load error #

Given #

  • Fixture file bad_two.atago.yaml is created.

Inputs #

Fixture bad_two.atago.yaml:

version: "1"
suite:
  name: bad two
scenarios:
  - name: which one is counted
    steps:
      - run:
          command: echo hi
      - assert:
          stdout:
            contains: h
            matches: i
            count: 1

When #

${atago} run bad_two.atago.yaml

Then #

  • exit code is 2
  • stderr contains exactly one countable matcher

Scenario: an unsatisfiable size range is a load error #

Given #

  • Fixture file bad_size.atago.yaml is created.

Inputs #

Fixture bad_size.atago.yaml:

version: "1"
suite:
  name: bad size
scenarios:
  - name: nothing can be both
    steps:
      - assert:
          file:
            path: out.txt
            min_size: 9
            max_size: 2

When #

${atago} run bad_size.atago.yaml

Then #

  • exit code is 2
  • stderr contains greater than max_size

skipped on Windows

Given #

  • Fixture file inner_link.atago.yaml is created.

Inputs #

Fixture inner_link.atago.yaml:

version: "1"
suite:
  name: inner link
scenarios:
  - name: the target is a link out of the workdir
    steps:
      # A fixture cannot plant an escaping symlink (the loader
      # refuses), so the program under test plants it — which is
      # exactly the threat the assertion has to survive.
      - run:
          shell: true
          command: "ln -s /etc/hosts link.txt"
      - assert:
          file:
            path: link.txt
            min_size: 1

When #

${atago} run inner_link.atago.yaml

Then #

  • exit code is 1
  • stdout contains symlink

Scenario: a size bound next to exists false is a load error #

Given #

  • Fixture file bad_absent.atago.yaml is created.

Inputs #

Fixture bad_absent.atago.yaml:

version: "1"
suite:
  name: bad absent
scenarios:
  - name: nothing can satisfy both
    steps:
      - assert:
          file:
            path: out.txt
            exists: false
            size: 0

When #

${atago} run bad_absent.atago.yaml

Then #

  • exit code is 2
  • stderr contains an absent file has no size

atago self-hosting / db runner #

Source: test/e2e/atago/db.atago.yaml

Scenario: query workflow (create, insert, select, row assert, value binding) passes #

Given #

  • Fixture file db.atago.yaml is created.

Inputs #

Fixture db.atago.yaml:

version: "1"
suite:
  name: inner-db
runners:
  store:
    type: db
    dsn: "sqlite::memory:"
scenarios:
  - name: seed then query with row assertions and binding
    steps:
      - query:
          runner: store
          sql: "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, role TEXT)"
      - query:
          runner: store
          sql: "INSERT INTO users (name, role) VALUES ('alice','admin'), ('bob','user')"
      - query:
          runner: store
          sql: "SELECT id, name, role FROM users ORDER BY id"
      - assert:
… (truncated, 23 more lines)

When #

${atago} run db.atago.yaml

Then #

  • exit code is 0
  • stdout contains passed

Scenario: a query against an undeclared runner fails validation (exit 2) #

Given #

  • Fixture file norunner.atago.yaml is created.

Inputs #

Fixture norunner.atago.yaml:

version: "1"
suite:
  name: inner-db
scenarios:
  - name: references a db runner that is not declared
    steps:
      - query:
          runner: missing
          sql: "SELECT 1"

When #

${atago} run norunner.atago.yaml

Then #

  • exit code is 2
  • stderr contains is not declared

Scenario: a hostaddr dsn is held to the network policy (exit 6) #

Given #

  • Fixture file hostaddr.atago.yaml is created.

Inputs #

Fixture hostaddr.atago.yaml:

version: "1"
suite:
  name: inner-hostaddr
  timeout: 10s
permissions:
  network:
    allow: ["db.allowed.example"]
runners:
  store:
    type: db
    driver: postgres
    dsn: "hostaddr=127.0.0.1 port=1 user=u dbname=app sslmode=disable"
scenarios:
  - name: reaches an address the policy never named
    steps:
      - query:
          runner: store
          sql: "SELECT 1"

When #

${atago} run hostaddr.atago.yaml

Then #

  • exit code is 6
  • stdout contains network policy denies host "127.0.0.1"

Scenario: a quoted host is compared to the allowlist without its quotes #

Given #

  • Fixture file quoted.atago.yaml is created.

Inputs #

Fixture quoted.atago.yaml:

version: "1"
suite:
  name: inner-quoted
  timeout: 10s
permissions:
  network:
    allow: ["127.0.0.1"]
runners:
  store:
    type: db
    driver: postgres
    dsn: "host='127.0.0.1' port=1 user=u dbname=app sslmode=disable"
scenarios:
  - name: the allowlisted host connects (and fails as a connection)
    steps:
      - query:
          runner: store
          sql: "SELECT 1"

When #

${atago} run quoted.atago.yaml

Then #

  • exit code is 4
  • stdout does not contain network policy denies

atago self-hosting / top-level defaults #

Source: test/e2e/atago/defaults.atago.yaml

Scenario: defaults.run.shell applies to every run step without repeating it #

Given #

  • Fixture file shell.atago.yaml is created.

Inputs #

Fixture shell.atago.yaml:

version: "1"
suite:
  name: shelled
defaults:
  run:
    shell: true
scenarios:
  - name: uses a shell builtin and env expansion
    env:
      WHO: world
    steps:
      - run:
          command: echo "hello $WHO"
      - assert:
          stdout:
            contains: hello world

When #

${atago} run shell.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: defaults.scenario.env is merged and an explicit scenario env wins #

Given #

  • Fixture file env.atago.yaml is created.

Inputs #

Fixture env.atago.yaml:

version: "1"
suite:
  name: enved
defaults:
  run:
    shell: true
  scenario:
    env:
      FROM_DEFAULT: base
      SHARED: default
scenarios:
  - name: sees the default and overrides one key
    env:
      SHARED: own
    steps:
      - run:
          command: echo "$FROM_DEFAULT-$SHARED"
      - assert:
          stdout:
            contains: base-own

When #

${atago} run env.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: defaults.run.sandbox_home governs a run step and a pty step alike (POSIX) #

skipped on Windows

Given #

  • Fixture file sandbox.atago.yaml is created.

Inputs #

Fixture sandbox.atago.yaml:

version: "1"
suite:
  name: shared-sandbox
defaults:
  run:
    shell: true
    sandbox_home: true
scenarios:
  - name: a run step writes and a pty step reads under one sandbox home
    steps:
      - run:
          command: 'mkdir -p "$XDG_CONFIG_HOME/mytool" && printf editor=vim > "$XDG_CONFIG_HOME/mytool/config"'
      - assert:
          exit_code: 0
      - pty:
          shell: true
          command: 'cat "$XDG_CONFIG_HOME/mytool/config"'
          session:
            - expect: 'editor=vim'
      - assert:
… (truncated, 6 more lines)

When #

${atago} run sandbox.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: an unsupported defaults field is a load-time error (exit 2) #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
defaults:
  run:
    command: echo nope
scenarios:
  - name: never runs
    steps:
      - run:
          command: echo hi

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains defaults.run.command is not supported

Scenario: defaults.run.env merges per key and a step env wins the collisions #

Given #

  • Fixture file env.atago.yaml is created.

Inputs #

Fixture env.atago.yaml:

version: "1"
suite:
  name: envmerge
defaults:
  run:
    shell: true
    env: {BASE: from-default, SHARED: default-value}
scenarios:
  - name: step env overrides one key and inherits the other
    steps:
      - run:
          command: "echo [$BASE][$SHARED]"
          env: {SHARED: step-value}
      - assert:
          stdout:
            contains: "[from-default][step-value]"

When #

${atago} run env.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: a step opts out of defaults.run.shell with an explicit shell false #

Given #

  • Fixture file optout.atago.yaml is created.

Inputs #

Fixture optout.atago.yaml:

version: "1"
suite:
  name: optout
defaults:
  run:
    shell: true
scenarios:
  - name: this step runs without a shell
    steps:
      - run:
          shell: false
          command: "echo literal-$HOME"
      - assert:
          stdout:
            contains: "literal-$HOME"

When #

${atago} run optout.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: a runner’s cwd beats defaults.run.cwd #

Given #

  • Fixture file cwd.atago.yaml is created.

Inputs #

Fixture cwd.atago.yaml:

version: "1"
suite:
  name: cwdprec
defaults:
  run:
    shell: true
    cwd: from-defaults
runners:
  located:
    type: cmd
    cwd: from-runner
scenarios:
  - name: the step lands in the runner's directory
    steps:
      - fixture: {file: from-defaults/keep, content: keep}
      - fixture: {file: from-runner/keep, content: keep}
      - run: {runner: located, command: "echo here > made.txt"}
      - assert: {exit_code: 0}
      - assert: {file: {path: from-runner/made.txt, exists: true}}
      - assert: {file: {path: from-defaults/made.txt, exists: false}}
… (truncated, 14 more lines)

When #

${atago} run cwd.atago.yaml

Then #

  • exit code is 0
  • stdout contains 3 passed

Scenario: defaults.scenario.only gates every scenario that states no gate #

Given #

  • Fixture file gated.atago.yaml is created.

Inputs #

Fixture gated.atago.yaml:

version: "1"
suite:
  name: gated
defaults:
  scenario:
    only:
      command: definitely-no-such-tool-xyz --version
scenarios:
  - name: inherits the file gate
    steps:
      - run: {command: "false"}
      - assert: {exit_code: 0}
  - name: also inherits it
    steps:
      - run: {command: "false"}
      - assert: {exit_code: 0}

When #

${atago} run gated.atago.yaml

Then #

  • exit code is 0
  • stdout contains 2 skipped

Scenario: a scenario’s own gate replaces the default rather than combining #

Given #

  • Fixture file own.atago.yaml is created.

Inputs #

Fixture own.atago.yaml:

version: "1"
suite:
  name: own
defaults:
  scenario:
    only:
      command: definitely-no-such-tool-xyz --version
scenarios:
  - name: states its own gate and runs
    only:
      command: "true"
    steps:
      - run: {command: "true"}
      - assert: {exit_code: 0}
  - name: takes the default and skips
    steps:
      - run: {command: "false"}
      - assert: {exit_code: 0}

When #

${atago} run own.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed, 1 skipped

Scenario: defaults.scenario.skip excludes every scenario that states no skip #

Given #

  • Fixture file skipped.atago.yaml is created.

Inputs #

Fixture skipped.atago.yaml:

version: "1"
suite:
  name: skipped
defaults:
  scenario:
    skip:
      command: "true"
scenarios:
  - name: inherits the file exclusion
    steps:
      - run: {command: "false"}
      - assert: {exit_code: 0}

When #

${atago} run skipped.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 skipped

atago self-hosting / deterministic runs #

Source: test/e2e/atago/deterministic.atago.yaml

Scenario: a read-only command satisfies the default check #

Given #

  • Fixture file rows.csv is created.

Inputs #

Fixture rows.csv:

id,name
1,alice

When #

cat rows.csv

Then #

  • exit code is 0
  • stdout contains alice

Scenario: the asserts and store still describe the first run #

When #

echo stable
# capture ${captured} from stdout
echo ${captured}

Then #

  • after echo stable:
    • stdout equals an exact value
  • after echo ${captured}:
    • stdout contains stable

Scenario: a stable failure is a stable answer #

When #

cat definitely-not-here.txt

Then #

  • exit code is not 0

Scenario: output that changes between runs fails with a diff #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: the command answers differently each time
    steps:
      - run:
          shell: true
          command: "echo $$; echo tail"
          deterministic: {}

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains deterministic: stdout identical, not deterministic, first run, later run

Scenario: a command that rewrites its own input is called out, not blamed #

Given #

  • Fixture file inner_mutating.atago.yaml is created.

Inputs #

Fixture inner_mutating.atago.yaml:

version: "1"
suite:
  name: inner mutating
scenarios:
  - name: each run appends to the file it prints
    steps:
      - fixture:
          file: log.txt
          content: "start\n"
      - run:
          shell: true
          command: "echo tick >> log.txt; cat log.txt"
          deterministic: {}

When #

${atago} run inner_mutating.atago.yaml

Then #

  • exit code is 1
  • stdout contains changed the workdir

Scenario: explain shows that the command runs more than once #

Given #

  • Fixture file shown.atago.yaml is created.

Inputs #

Fixture shown.atago.yaml:

version: "1"
suite:
  name: shown
scenarios:
  - name: repeated
    steps:
      - run:
          command: echo hi
          deterministic:
            runs: 3
            compare: [stdout]

When #

${atago} explain shown.atago.yaml

Then #

  • exit code is 0
  • stdout contains deterministic: stdout compared across 3 runs

Scenario: deterministic next to retry is a load error #

Given #

  • Fixture file bad_retry.atago.yaml is created.

Inputs #

Fixture bad_retry.atago.yaml:

version: "1"
suite:
  name: bad retry
scenarios:
  - name: contradictory
    steps:
      - run:
          command: echo hi
          deterministic: {}
          retry:
            times: 3
            until:
              exit_code: 0

When #

${atago} run bad_retry.atago.yaml

Then #

  • exit code is 2
  • stderr contains cannot be combined with retry

Scenario: a single run and an unknown observable are load errors #

Given #

  • Fixture file bad_runs.atago.yaml is created.
  • Fixture file bad_compare.atago.yaml is created.

Inputs #

Fixture bad_runs.atago.yaml:

version: "1"
suite:
  name: bad runs
scenarios:
  - name: one run proves nothing
    steps:
      - run:
          command: echo hi
          deterministic:
            runs: 1

Fixture bad_compare.atago.yaml:

version: "1"
suite:
  name: bad compare
scenarios:
  - name: nothing to compare there
    steps:
      - run:
          command: echo hi
          deterministic:
            compare: [workdir]

When #

${atago} run bad_runs.atago.yaml
${atago} run bad_compare.atago.yaml

Then #

  • after ${atago} run bad_runs.atago.yaml:
    • exit code is 2
    • stderr contains must be at least 2
  • after ${atago} run bad_compare.atago.yaml:
    • exit code is 2
    • stderr contains unknown observable

atago self-hosting / dir assertion #

Source: test/e2e/atago/dir.atago.yaml

Scenario: directory/tree assertions cover a multi-file generator #

skipped on Windows

When #

mkdir -p site/assets && printf '<html>' > site/index.html && printf '<html>' > site/about.html && printf 'body{}' > site/assets/app.css

Then #

  • exit code is 0
  • dir site exists, contains index.html, contains about.html, contains assets/app.css, does not contain secret.txt, has 3 entries, has >= 1 entry, has <= 10 entries, matches glob *.html

Scenario: a missing directory can be asserted absent #

Then #

  • dir never-created does not exist

Scenario: a dangling symlink is a present directory entry (membership uses Lstat) #

skipped on Windows

When #

mkdir -p linkdir && ln -s /nonexistent-target-xyz linkdir/planted

Then #

  • exit code is 0
  • dir linkdir contains planted, does not contain never-planted

Scenario: a symlinked directory reads the same in every dir mode #

skipped on Windows

When #

mkdir -p releases/v2/assets && echo bin > releases/v2/app.bin && echo css > releases/v2/assets/app.css && ln -s releases/v2 latest

Then #

  • exit code is 0
  • dir latest contains app.bin
  • dir latest contains assets/app.css, (recursive)
  • dir latest has 2 entries, (recursive)

Scenario: a dir path that resolves out of the workdir is refused #

skipped on Windows

Given #

  • Fixture file escaping.atago.yaml is created.

Inputs #

Fixture escaping.atago.yaml:

version: "1"
suite:
  name: escaping dir assert
scenarios:
  - name: read outside the workdir through a link
    steps:
      - run:
          shell: true
          command: ln -s /etc escape
      - assert:
          dir:
            path: escape
            contains: [hosts]

When #

ln -s /etc escape
${atago} run escaping.atago.yaml

Then #

  • after ln -s /etc escape:
    • exit code is 0
  • after ${atago} run escaping.atago.yaml:
    • exit code is 1
    • stdout contains escapes the scenario workdir

Scenario: a failed dir assert lists what the directory actually holds #

Given #

  • Fixture file listing.atago.yaml is created.

Inputs #

Fixture listing.atago.yaml:

version: "1"
suite:
  name: listing
scenarios:
  - name: the generator produced the wrong tree
    steps:
      - fixture: {file: out/index.html, content: "x"}
      - fixture: {file: out/app.js, content: "x"}
      - run: {command: echo built}
      - assert:
          dir:
            path: out
            contains: [style.css]

When #

${atago} run listing.atago.yaml

Then #

  • exit code is 1
  • stdout contains app.js, index.html

Scenario: a failed count assert lists the entries it counted #

Given #

  • Fixture file counted.atago.yaml is created.

Inputs #

Fixture counted.atago.yaml:

version: "1"
suite:
  name: counted
scenarios:
  - name: the count is off
    steps:
      - fixture: {file: out/only.txt, content: "x"}
      - run: {command: echo built}
      - assert:
          dir:
            path: out
            count: 4

When #

${atago} run counted.atago.yaml

Then #

  • exit code is 1
  • stdout contains only.txt, has 1 entry

Scenario: a failed recursive assert lists the walked tree #

Given #

  • Fixture file walked.atago.yaml is created.

Inputs #

Fixture walked.atago.yaml:

version: "1"
suite:
  name: walked
scenarios:
  - name: the nested path is missing
    steps:
      - fixture: {file: out/assets/app.css, content: "x"}
      - fixture: {file: out/index.html, content: "x"}
      - run: {command: echo built}
      - assert:
          dir:
            path: out
            recursive: true
            contains: [assets/missing.css]

When #

${atago} run walked.atago.yaml

Then #

  • exit code is 1
  • stdout contains assets/app.css, index.html

Scenario: a dir assertion on a file path says the path is a file #

Given #

  • Fixture file mistaken.atago.yaml is created.

Inputs #

Fixture mistaken.atago.yaml:

version: "1"
suite: {name: mistaken}
scenarios:
  - name: out is a file, not a directory
    steps:
      - fixture: {file: out, content: "report\n"}
      - run: {command: echo built}
      - assert:
          dir:
            path: out
            exists: true

When #

${atago} run mistaken.atago.yaml

Then #

  • exit code is 1
  • stdout contains exists but is a regular file, use a file: assertion

Scenario: a named pipe in the tree does not block the walk #

skipped on Windows

Given #

  • Fixture file treespec.atago.yaml is created.

Inputs #

Fixture treespec.atago.yaml:

version: "1"
suite: {name: treesnap}
scenarios:
  - name: snapshot a tree holding a pipe
    steps:
      - run: {shell: true, command: "mkdir out && mkfifo out/pipe && printf 'x' > out/real.txt"}
      - assert: {dir: {path: out, snapshot: pipe_tree}}

When #

mkdir out && mkfifo out/pipe && printf 'x' > out/real.txt
${atago} run --update-snapshots treespec.atago.yaml
cat pipe_tree
${atago} run treespec.atago.yaml

Then #

  • after mkdir out && mkfifo out/pipe && printf 'x' > out/real.txt:
    • dir out contains pipe, contains real.txt, has 1 entry, (recursive)
  • after ${atago} run --update-snapshots treespec.atago.yaml:
    • exit code is 0
  • after cat pipe_tree:
    • stdout contains fifo pipe, file real.txt sha256:
  • after ${atago} run treespec.atago.yaml:
    • exit code is 0

Scenario: a file assertion on a named pipe fails instead of hanging #

skipped on Windows

Given #

  • Fixture file piped.atago.yaml is created.

Inputs #

Fixture piped.atago.yaml:

version: "1"
suite: {name: piped}
scenarios:
  - name: the tool left a pipe where a file was expected
    steps:
      - run: {shell: true, command: "mkfifo out.txt"}
      - assert: {file: {path: out.txt, equals: "data"}}

When #

${atago} run piped.atago.yaml

Then #

  • exit code is 1
  • stdout contains is a named pipe, not a regular file

atago self-hosting / recursive dir asserts + tree snapshots #

Source: test/e2e/atago/dir_tree.atago.yaml

Scenario: record, compare green, then a mutation names the changed paths #

Given #

  • Fixture file inner.atago.yaml is created.
  • Fixture file inner_mutated.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: tree matches its golden
    steps:
      - fixture:
          file: site/hugo.toml
          content: "baseURL = 'x'\n"
      - fixture:
          file: site/content/posts/hello.md
          content: "# hello\n"
      - assert:
          dir:
            path: site
            snapshot: snapshots/site_tree.txt

Fixture inner_mutated.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: tree matches its golden
    steps:
      - fixture:
          file: site/hugo.toml
          content: "baseURL = 'x'\n"
      - fixture:
          file: site/content/posts/hello.md
          content: "# CHANGED\n"
      - fixture:
          file: site/extra.txt
          content: "new\n"
      - assert:
          dir:
            path: site
            snapshot: snapshots/site_tree.txt

When #

${atago} run --update-snapshots inner.atago.yaml
${atago} run inner.atago.yaml
${atago} run inner_mutated.atago.yaml

Then #

  • after ${atago} run --update-snapshots inner.atago.yaml:
    • exit code is 0
    • file snapshots/site_tree.txt contains dir content, file hugo.toml sha256:
  • after ${atago} run inner.atago.yaml:
    • exit code is 0
  • after ${atago} run inner_mutated.atago.yaml:
    • exit code is 1
    • stdout contains added: file extra.txt, changed: content/posts/hello.md

Scenario: recursive matchers and ignore globs walk the tree #

Given #

  • Fixture file out/a/deep/nested.md is created.
  • Fixture file out/top.txt is created.
  • Fixture file out/noise.log is created.

Inputs #

Fixture out/a/deep/nested.md:

nested

Fixture out/top.txt:

top

Fixture out/noise.log:

noise

Then #

  • dir out contains a/deep/nested.md, has 2 entries, matches glob *.md, (recursive), ignoring *.log
  • dir out does not contain a/deep/missing.md, (recursive)

Scenario: a recursive count failure speaks in files, not entries #

Given #

  • Fixture file inner_count.atago.yaml is created.

Inputs #

Fixture inner_count.atago.yaml:

version: "1"
suite:
  name: inner count
scenarios:
  - name: tree file count misses
    steps:
      - fixture:
          file: out/kept.txt
          content: "kept\n"
      - fixture:
          file: out/sub/nested.txt
          content: "nested\n"
      - assert:
          dir:
            path: out
            recursive: true
            count: 5

When #

${atago} run inner_count.atago.yaml

Then #

  • exit code is 1
  • stdout contains assert dir "out" file count, directory "out" has 2 files, expected exactly 5 files in the tree, does not contain entries

Scenario: combining snapshot with matchers is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: over-specified
    steps:
      - run: {command: echo hi}
      - assert:
          dir:
            path: out
            snapshot: tree.txt
            contains: [a.txt]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains snapshot cannot be combined

atago self-hosting / doc #

Source: test/e2e/atago/doc.atago.yaml

Scenario: doc generates Markdown to a file #

Given #

  • Fixture file target.atago.yaml is created.

Inputs #

Fixture target.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: greet
    steps:
      - run:
          command: echo hello
      - assert:
          stdout:
            contains: hello

When #

${atago} doc --out specs.md target.atago.yaml

Then #

  • exit code is 0
  • file specs.md contains # atago Behavior Specs
  • file specs.md contains ### Scenario: greet

Scenario: doc writes Markdown to stdout without –out #

Given #

  • Fixture file t2.atago.yaml is created.

Inputs #

Fixture t2.atago.yaml:

version: "1"
suite:
  name: sample2
scenarios:
  - name: g2
    steps:
      - run:
          command: echo hi
      - assert:
          exit_code: 0

When #

${atago} doc t2.atago.yaml

Then #

  • exit code is 0
  • stdout contains ## sample2

Scenario: doc emits a summary, table of contents, and input previews #

Given #

  • Fixture file rich.atago.yaml is created.

Inputs #

Fixture rich.atago.yaml:

version: "1"
suite:
  name: rich
scenarios:
  - name: seeded query
    tags: [smoke]
    steps:
      - fixture:
          file: data.csv
          content: "id,name\n1,ada\n"
      - run:
          command: cat data.csv
      - assert:
          exit_code: 0

When #

${atago} doc rich.atago.yaml

Then #

  • exit code is 0
  • stdout contains ## Summary, 1 suite · 1 scenario, ## Contents, (#scenario-seeded-query), #### Inputs, id,name

Scenario: doc –split-by-spec writes one file per spec and an index #

Given #

  • Fixture file one.atago.yaml is created.
  • Fixture file two.atago.yaml is created.

Inputs #

Fixture one.atago.yaml:

version: "1"
suite:
  name: suite-one
scenarios:
  - name: a
    steps:
      - run: {command: "true"}
      - assert: {exit_code: 0}

Fixture two.atago.yaml:

version: "1"
suite:
  name: suite-two
scenarios:
  - name: b
    steps:
      - run: {command: "true"}
      - assert: {exit_code: 0}

When #

${atago} doc --split-by-spec --out-dir generated one.atago.yaml two.atago.yaml

Then #

  • exit code is 0
  • file generated/one.md exists
  • file generated/two.md exists
  • file generated/index.md contains atago Behavior Specs — Index
  • file generated/index.md contains (one.md)

Generated artifacts #

  • generated/one.md
  • generated/two.md

Scenario: doc –split-by-spec requires –out-dir #

Given #

  • Fixture file solo.atago.yaml is created.

Inputs #

Fixture solo.atago.yaml:

version: "1"
suite:
  name: solo
scenarios:
  - name: only
    steps:
      - run: {command: "true"}
      - assert: {exit_code: 0}

When #

${atago} doc --split-by-spec solo.atago.yaml

Then #

  • exit code is 3
  • stderr contains requires --out-dir

Scenario: doc renders suite and scenario descriptions verbatim #

Given #

  • Fixture file described.atago.yaml is created.

Inputs #

Fixture described.atago.yaml:

version: "1"
suite:
  name: described
  description: |
    What this suite guarantees for its users.

    Second paragraph, ${not_a_variable} included.
scenarios:
  - name: greets
    description: "Guards the regression from #123."
    steps:
      - run: {command: echo hi}
      - assert: {exit_code: 0}

When #

${atago} doc described.atago.yaml
${atago} run described.atago.yaml

Then #

  • after ${atago} doc described.atago.yaml:
    • exit code is 0
    • stdout contains What this suite guarantees for its users.
    • stdout contains Second paragraph, ${not_a_variable} included.
    • stdout contains Guards the regression from #123.
  • after ${atago} run described.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: doc renders every matcher an assertion sets #

Given #

  • Fixture file matchers.atago.yaml is created.

Inputs #

Fixture matchers.atago.yaml:

version: "1"
suite:
  name: matchers
scenarios:
  - name: composed stream
    steps:
      - run:
          command: echo hello
      - assert:
          stdout:
            contains: hello
            not_contains: goodbye
  - name: line scoped
    steps:
      - run:
          shell: true
          command: printf 'one\ntwo\n'
      - assert:
          stdout:
            line: 2
… (truncated, 17 more lines)

When #

${atago} doc matchers.atago.yaml

Then #

  • exit code is 0
  • stdout contains stdout contains , , does not contain , stdout line , equals an exact value, is executable, does not contain
  • stdout does not contain is checked

Scenario: a spec whose matchers doc renders still runs green #

skipped on Windows

Given #

  • Fixture file matchers.atago.yaml is created.

Inputs #

Fixture matchers.atago.yaml:

version: "1"
suite:
  name: matchers
scenarios:
  - name: composed stream
    steps:
      - run:
          command: echo hello
      - assert:
          stdout:
            contains: hello
            not_contains: goodbye
  - name: line scoped
    steps:
      - run:
          shell: true
          command: printf 'one\ntwo\n'
      - assert:
          stdout:
            line: 2
… (truncated, 17 more lines)

When #

${atago} run matchers.atago.yaml

Then #

  • exit code is 0
  • stdout contains 3 passed

Scenario: doc renders the suite lifecycle blocks #

Given #

  • Fixture file lifecycle.atago.yaml is created.

Inputs #

Fixture lifecycle.atago.yaml:

version: "1"
suite:
  name: lifecycle
  setup:
    - run:
        command: echo build
    - service:
        name: relay
        command: echo serve
        ready:
          delay: 10ms
  teardown:
    - run:
        command: echo purge
scenarios:
  - name: trivial
    steps:
      - run: {command: echo hi}
      - assert: {exit_code: 0}

When #

${atago} doc lifecycle.atago.yaml

Then #

  • exit code is 0
  • stdout contains ### Suite setup (runs once before any scenario), echo build, # start service relay: echo serve, ### Suite teardown (always runs after the last scenario), echo purge

Scenario: doc keeps teardown assertions and states the suite guarantees #

Given #

  • Fixture file narrative.atago.yaml is created.

Inputs #

Fixture narrative.atago.yaml:

version: "1"
suite:
  name: narrative
  setup:
    - fixture: {file: seed.json, content: "{}"}
permissions:
  network:
    allow: [api.example.com]
secrets: [API_TOKEN]
scenarios:
  - name: cleans up after itself
    steps:
      - run: {shell: true, command: "echo body > out.txt"}
      - assert: {exit_code: 0}
    teardown:
      - assert: {file: {path: out.txt, exists: true}}

When #

${atago} doc narrative.atago.yaml

Then #

  • exit code is 0
  • stdout contains Network policy: egress is allowed only to `api.example.com`., Secrets declared: `API_TOKEN`., ### Suite setup (runs once before any scenario), # write fixture seed.json, #### Finally (teardown, always runs), # expect file `out.txt` exists

atago self-hosting / duration assertion #

Source: test/e2e/atago/duration.atago.yaml

Scenario: a fast step passes a generous upper bound #

When #

${atago} version

Then #

  • exit code is 0
  • completes in under 60s

Scenario: an impossible bound fails and shows the measured duration #

Given #

  • Fixture file slow.atago.yaml is created.

Inputs #

Fixture slow.atago.yaml:

version: "1"
suite:
  name: slow
scenarios:
  - name: cannot beat 1ns
    steps:
      - run:
          command: ${atago} version
      - assert:
          duration:
            lt: 1ns

When #

${atago} run slow.atago.yaml

Then #

  • exit code is 1
  • stdout contains assert duration < 1ns, orders of magnitude

Scenario: a deliberate wait satisfies a lower bound #

skipped on Windows

When #

sleep 0.2

Then #

  • exit code is 0
  • completes in under 60s and in at least 100ms

Scenario: a duration assert with no preceding step is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: no step to measure
    steps:
      - assert:
          duration: {lt: 2s}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains requires an immediately preceding

atago self-hosting / edge cases #

Source: test/e2e/atago/edge.atago.yaml

Scenario: JSON assertion on empty stdout reports an empty stream #

Given #

  • Fixture file empty.atago.yaml is created.

Inputs #

Fixture empty.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: json on empty
    steps:
      - run:
          shell: true
          command: "exit 0"
      - assert:
          stdout:
            json:
              path: "$.x"
              equals: 1

When #

${atago} run empty.atago.yaml

Then #

  • exit code is 1
  • stdout contains was empty

Scenario: an unsupported matcher is a parse error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: csv matcher
    steps:
      - run:
          shell: true
          command: echo hi
      - assert:
          stdout:
            csv:
              column: name

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2

Scenario: a mixed valid+invalid run reads FAILED and counts the dropped spec #

Given #

  • Fixture file good.atago.yaml is created.
  • Fixture file broken.atago.yaml is created.

Inputs #

Fixture good.atago.yaml:

version: "1"
suite:
  name: good
scenarios:
  - name: echo works
    steps:
      - run: {shell: true, command: echo hi}
      - assert: {exit_code: 0}

Fixture broken.atago.yaml:

version: "1"
suite:
  name: broken
scenarios:
  - name: uses an unknown field
    steps:
      - nonsense_field: {command: echo nope}

When #

${atago} run good.atago.yaml broken.atago.yaml

Then #

  • exit code is 2
  • stdout contains 1 spec failed to load
  • stdout does not contain PASSED

Scenario: a snapshot update error names the snapshot command, not run #

When #

${atago} snapshot update no-such-spec.atago.yaml

Then #

  • exit code is 3
  • stderr contains atago snapshot update: , cannot access, does not contain atago run:

Scenario: a json assertion on malformed input fails cleanly, without a crash #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: badjson}
scenarios:
  - name: json on broken bytes
    steps:
      - fixture: {file: broken.json, content: '{"":f,"":0 0'}
      - assert: {file: {path: broken.json, json: {path: "$.x", equals: 1}}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 1
  • stdout contains not valid JSON

atago self-hosting / workdir + scenario env + not_contains #

Source: test/e2e/atago/env_workdir.atago.yaml

Scenario: run.stdout_to redirects stdout to a workdir file without a shell #

When #

printf marked

Then #

  • exit code is 0
  • file marker.txt contains marked

Generated artifacts #

  • marker.txt

Scenario: scenario env is shared by every run step and overridable per step #

Given #

  • Environment variables are set: OVERRIDE_ME.

When #

printf '%s\n' "$SHARED"
printf '%s\n' "$OVERRIDE_ME"

Then #

  • after printf '%s\n' "$SHARED":
    • stdout equals an exact value
  • after printf '%s\n' "$OVERRIDE_ME":
    • stdout equals an exact value

Scenario: scenario env can reference ${workdir} for isolated paths #

When #

printf '%s\n' "$ISO_HOME"

Then #

  • stdout contains /home
  • stdout does not contain ${workdir}/home/extra

Scenario: file not_contains passes when the substring is absent #

Given #

  • Fixture file data.txt is created.

Inputs #

Fixture data.txt:

alpha
beta

Then #

  • file data.txt does not contain gamma

Scenario: not_contains fails when the substring is present #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: stdout unexpectedly contains hello
    steps:
      - run:
          command: echo hello world
      - assert:
          stdout:
            not_contains: hello

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains unexpectedly present

Scenario: a shell metacharacter without shell is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: forgot shell true
    steps:
      - run:
          command: echo hi > out.txt

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains shell is not enabled, shell: true, stdout_to

atago self-hosting / every diagnostic code #

Source: test/e2e/atago/error_codes.atago.yaml

Scenario: ATG2001 is a spec file that cannot be read #

skipped on Windows

Given #

  • Fixture file broken.atago.yaml is created.

When #

${atago} run .

Then #

  • exit code is 2
  • stderr contains ATG2001

Scenario: ATG2002 is a spec file with no YAML document in it #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

# only a comment

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2002

Scenario: ATG2003 is a document that is not valid YAML #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
  suite: {name: x}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2003

Scenario: ATG2004 is an explicit YAML tag #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: !!str x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2004

Scenario: ATG2005 is a key the schema does not define #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenariosss: []

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2005, unknown field "scenariosss"

Scenario: ATG2006 is a value written in a shape its key cannot take #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{assert: {stdout: hi}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2006

Scenario: ATG2010 is an unsupported spec format version #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "9"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2010

Scenario: ATG2101 is a step that sets no action #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2101

Scenario: ATG2102 is a step that sets more than one action #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
        fixture: {file: f, content: c}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2102

Scenario: ATG2103 is a pair of keys that contradict each other #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {file: {path: f, size: 1, snapshot: s}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2103

Scenario: ATG2104 is a group that takes exactly one member and got two #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - pty: {command: echo, session: [{expect: a, send: b}]}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2104

Scenario: ATG2105 is a real key in a position where it means nothing #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
defaults: {run: {command: echo}}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2105

Scenario: ATG2106 is a block at the wrong level of the spec #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - service: {name: s, command: "sleep 1"}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2106

Scenario: ATG2107 is an assertion with no step to describe #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {screen: {contains: hi}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2107

Scenario: ATG2108 is a key whose companion key is not set #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {image: {path: f, max_diff: 0.5}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2108

Scenario: ATG2201 is a required key that is absent #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2201

Scenario: ATG2202 is a list that must hold an entry and holds none #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: []

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2202

Scenario: ATG2203 is a group that needs at least one member and got none #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {changes: {}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2203

Scenario: ATG2204 is a key present with an empty value #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {file: {path: f, equals_file: ""}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2204

Scenario: ATG2301 is a duration atago cannot read #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x, timeout: "soon"}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2301

Scenario: ATG2302 is a negative value where a negative means nothing #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x, timeout: "-1s"}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2302

Scenario: ATG2303 is a value that must be positive and is not #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    services: [{name: s, command: "sleep 1", max_log_bytes: -1}]
    steps:
      - run: {command: echo}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2303

Scenario: ATG2304 is a number outside the range its key accepts #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - pty: {command: echo, session: [{expect: a}]}
      - assert: {screen: {attrs: [{text: hi, bold: true, row: -1}]}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2304

Scenario: ATG2305 is a regular expression that does not compile #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {matches: "["}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2305

Scenario: ATG2306 is a glob pattern that does not parse #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {dir: {path: ".", glob: "["}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2306

Scenario: ATG2307 is a value outside a closed vocabulary #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, skip: {os: macos}, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2307

Scenario: ATG2308 is a range nothing could satisfy #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {file: {path: f, min_size: 10, max_size: 5}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2308

Scenario: ATG2309 is an absolute path where a relative one is required #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {changes: {created: ["/tmp/x"]}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2309

Scenario: ATG2310 is a path that climbs out of the workdir #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {changes: {created: ["../x"]}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2310

Scenario: ATG2311 is a control character in a name #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: "x\ty"}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2311

Scenario: ATG2312 is a matcher that could never fail #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {contains: ""}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2312

Scenario: ATG2313 is a value in the wrong notation #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - fixture: {file: f, content: c, mode: "99"}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2313

Scenario: ATG2401 is a runner the spec never declared #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - http: {runner: api, method: GET, path: /}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2401

Scenario: ATG2402 is a declared runner of the wrong type #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
runners: {api: {type: db, dsn: "sqlite://:memory:"}}
scenarios:
  - name: a
    steps:
      - http: {runner: api, method: GET, path: /}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2402

Scenario: ATG2403 is a reference to something the spec never declared #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - signal: {service: ghost, signal: TERM}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2403

Scenario: ATG2404 is a stored value shadowing a built-in #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - store: {name: workdir, from: {stdout: {matches: "x"}}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2404

Scenario: ATG2405 is a manifest path that resolves to nothing #

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

fixtures_dir: nowhere

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2405

Scenario: ATG2501 is two scenarios sharing a name #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - {name: a, steps: [{run: {command: echo}}]}
  - {name: a, steps: [{run: {command: echo}}]}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2501

Scenario: ATG2502 is a set listing the same entry twice #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {exit_code: {in: [0, 0]}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2502

Scenario: a code is added to the message rather than replacing it #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr matches /ATG2201: suite\.name is required/

Scenario: several problems each report their own code in one pass #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "9"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {matches: "["}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2010, ATG2305

Scenario: ATG3001 is a subcommand atago does not have #

When #

${atago} frobnicate

Then #

  • exit code is 3
  • stderr contains ATG3001

Scenario: ATG3001 is also atago with no subcommand at all #

When #

${atago}

Then #

  • exit code is 3
  • stderr contains ATG3001

Scenario: ATG3002 is a subcommand called in a shape it does not accept #

When #

${atago} snapshot

Then #

  • exit code is 3
  • stderr contains ATG3002

Scenario: ATG3101 is an option atago does not define #

When #

${atago} run --definitely-not-a-flag .

Then #

  • exit code is 3
  • stderr contains ATG3101, flag provided but not defined, Usage: atago run

Scenario: ATG3102 is an option given a value it does not accept #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run --report jnit ok.atago.yaml

Then #

  • exit code is 3
  • stderr contains ATG3102

Scenario: ATG3103 is an option whose companion option is not set #

When #

${atago} doc --split-by-spec .

Then #

  • exit code is 3
  • stderr contains ATG3103

Scenario: ATG3104 is two options that contradict each other #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run --repeat 2 --retry-failed 1 ok.atago.yaml

Then #

  • exit code is 3
  • stderr contains ATG3104

Scenario: ATG3105 is a numeric option outside its range #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run --parallel -1 ok.atago.yaml

Then #

  • exit code is 3
  • stderr contains ATG3105

Scenario: ATG3201 is a path that cannot be reached #

When #

${atago} run ./nowhere

Then #

  • exit code is 3
  • stderr contains ATG3201

Scenario: ATG3202 is a directory holding no specs #

When #

${atago} run .

Then #

  • exit code is 3
  • stderr contains ATG3202

Scenario: ATG3203 is a selection that matched nothing under –ci #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run --ci --tag no-such-tag ok.atago.yaml

Then #

  • exit code is 3
  • stderr contains ATG3203

Scenario: ATG3204 is a rerun whose recorded failures no longer exist #

Given #

  • Fixture file ok.atago.yaml is created.
  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: before
    steps:
      - run: {command: "false"}
      - assert: {exit_code: 0}

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: after
    steps:
      - run: {command: echo}

When #

${atago} run ok.atago.yaml
${atago} run --rerun-failed ok.atago.yaml

Then #

  • after ${atago} run ok.atago.yaml:
    • exit code is 1
  • after ${atago} run --rerun-failed ok.atago.yaml:
    • exit code is 3
    • stderr contains ATG3204

Scenario: ATG3205 is a write that would replace an existing file #

Given #

  • Fixture file taken.atago.yaml is created.

Inputs #

Fixture taken.atago.yaml:

already here

When #

${atago} init taken.atago.yaml

Then #

  • exit code is 3
  • stderr contains ATG3205

Scenario: ATG3206 is a destination that cannot be written #

Given #

  • Fixture file occupied is created.
  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture occupied:

not a directory

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run --artifacts-dir occupied ok.atago.yaml

Then #

  • exit code is 3
  • stderr contains ATG3206

Scenario: ATG3207 is atago’s recorded state failing to load #

Given #

  • Fixture file .atago/last-failed.json is created.
  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture .atago/last-failed.json:

not json at all

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run --rerun-failed ok.atago.yaml

Then #

  • exit code is 3
  • stderr contains ATG3207

Scenario: ATG6001 is a request to a host the network policy denies #

Given #

  • Fixture file denied.atago.yaml is created.

Inputs #

Fixture denied.atago.yaml:

version: "1"
suite: {name: denied}
permissions:
  network:
    allow:
      - allowed.example
runners:
  api:
    type: http
    base_url: http://127.0.0.1:1
scenarios:
  - name: reaches elsewhere
    steps:
      - http: {runner: api, method: GET, path: /ping}

When #

${atago} run denied.atago.yaml

Then #

  • exit code is 6
  • stdout contains ATG6001

Scenario: ATG4001 is a command line that cannot be split into arguments #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - run: {command: "echo 'unclosed"}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4001

Scenario: ATG4002 is a program that could not be started #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - run: {command: definitely-no-such-binary-xyz}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4002

Scenario: ATG4003 is an environment the step needs that could not be prepared #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - run: {command: cat, stdin: {file: never-created.txt}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4003

Scenario: ATG4004 is a file the step depends on that cannot be read #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"

suite:
  name: x
  setup:
    - mock_server: {name: m, routes: [{method: POST, path: /p, status: 200}]}
runners: {api: {type: http, base_url: "${m.url}"}}
scenarios:
  - name: a
    steps:
      - http: {runner: api, method: POST, path: /p, body_file: never-created.bin}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4004

Scenario: ATG4005 is a command run beside a terminal session that failed #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - pty: {command: cat, timeout: 20s, session: [{exec: {command: "false"}}]}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4005

Scenario: ATG4101 is a step that outlasted its timeout #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - pty: {command: cat, timeout: 20s, session: [{exec: {command: "sleep 5", timeout: 300ms}}]}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4101

Scenario: ATG4102 is a service that never became ready #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    services:
      - {name: s, command: "sleep 30", ready: {port: "127.0.0.1:59991", timeout: 1s}}
    steps:
      - run: {command: echo}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4102

Scenario: ATG4201 is a peer that could not be reached #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

runners: {api: {type: http, base_url: "http://127.0.0.1:1"}}
scenarios:
  - name: a
    steps:
      - http: {runner: api, method: GET, path: /}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4201

Scenario: ATG4202 is a runner missing what it needs to connect #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

runners: {r: {type: ssh, host: "127.0.0.1:22", user: u, password: p}}
scenarios:
  - name: a
    steps:
      - run: {runner: r, command: echo}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4202

Scenario: ATG4203 is a peer that was reached and refused the request #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

runners: {db: {type: db, dsn: "sqlite://:memory:"}}
scenarios:
  - name: a
    steps:
      - query: {runner: db, sql: "SELECT * FROM no_such_table"}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4203

Scenario: ATG4204 is an address atago cannot make sense of #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

runners: {db: {type: db, dsn: "weird://x"}}
scenarios:
  - name: a
    steps:
      - query: {runner: db, sql: "SELECT 1"}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4204

Scenario: ATG4301 is a service that failed before it became ready #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    services:
      - {name: s, command: "true", ready: {port: "127.0.0.1:59992", timeout: 5s}}
    steps:
      - run: {command: echo}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4301

Scenario: ATG4302 is a service addressed after it exited #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    services:
      - {name: s, command: "sleep 1", ready: {delay: 100ms}}
    steps:
      - run: {command: "sleep 2"}
      - signal: {service: s, signal: TERM}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4302

Scenario: ATG4403 is output that could not be captured #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - run: {command: "sleep 4 &", shell: true, timeout: 30s}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4403

Scenario: ATG4406 is input the program is not set up to receive #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - pty: {command: cat, timeout: 20s, session: [{send: {mouse: {row: 1, col: 1}}}]}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4406

Scenario: ATG4501 is a store step with no result behind it #

Given #

  • Stub HTTP server api serves 1 canned route(s) at ${api.url} and records every request (#24).
  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
runners:
  api:
    type: http
    base_url: ${api.url}

scenarios:
  - name: a
    steps:
      - http: {runner: api, method: GET, path: /v1/ok}
      - run: {command: echo}
      - store: {name: v, from: {body: {trim: true}}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4501

Scenario: ATG4502 is a store selector that found nothing #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - store: {name: v, from: {stdout: {matches: "no-such-text-anywhere"}}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4502

Scenario: ATG4505 is a variable the step expands that is not defined #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}

scenarios:
  - name: a
    steps:
      - pty:
          command: cat
          timeout: 20s
          session:
            - send: "${env:ATAGO_DEFINITELY_UNSET_XYZ}"

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4505

Scenario: atago explain prints what a code means, offline #

When #

${atago} explain ATG2201

Then #

  • exit code is 0
  • stdout contains ATG2201, a required key is missing, Fix, Exits 2.

Scenario: atago explain refuses a code nobody assigned, and suggests one #

When #

${atago} explain ATG2999

Then #

  • exit code is 3
  • stderr contains ATG3102, unknown diagnostic code, did you mean

Scenario: the JSON report carries the code as a field #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: definitely-no-such-binary-xyz}

When #

${atago} run --report json bad.atago.yaml

Then #

  • exit code is 4
  • stdout at $.suites[0].failures[0].code equals ATG4002

Scenario: an assertion failure carries no code in the JSON report #

Given #

  • Fixture file fails.atago.yaml is created.

Inputs #

Fixture fails.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: "false"}
      - assert: {exit_code: 0}

When #

${atago} run --report json fails.atago.yaml

Then #

  • exit code is 1
  • stdout does not contain "code"

Scenario: a wrapped error reports one code, not two #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    services:
      - {name: s, command: "sleep 30", ready: {port: "127.0.0.1:59993", timeout: 1s}}
    steps:
      - run: {command: echo}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4102, does not contain ATG4301: service

Scenario: an assertion with no producing step is a load-time error #

Given #

  • Fixture file nocontext.atago.yaml is created.
  • Fixture file withcontext.atago.yaml is created.

Inputs #

Fixture nocontext.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: asserts an http status after a command
    steps:
      - run: {shell: true, command: "true"}
      - assert: {status: 200}

Fixture withcontext.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: asserts stdout after the command that produced it
    steps:
      - run: {shell: true, command: "echo ready"}
      - assert: {stdout: {contains: ready}}

When #

${atago} run nocontext.atago.yaml
${atago} run withcontext.atago.yaml

Then #

  • after ${atago} run nocontext.atago.yaml:
    • exit code is 2
    • stderr contains ATG2107, assert.status requires a preceding http step
  • after ${atago} run withcontext.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: a store with no producing step is a load-time error #

Given #

  • Fixture file nostore.atago.yaml is created.

Inputs #

Fixture nostore.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: stores a response header after a command
    steps:
      - run: {shell: true, command: "true"}
      - store: {name: token, from: {header: X-Token}}

When #

${atago} run nostore.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2107, store.from.header requires a preceding http step

Scenario: a suite block read that can never be fed is a load-time error #

Given #

  • Fixture file suiteread.atago.yaml is created.
  • Fixture file suiteok.atago.yaml is created.

Inputs #

Fixture suiteread.atago.yaml:

version: "1"
suite:
  name: inner
  teardown:
    - assert: {status: 200}
scenarios:
  - name: a
    steps:
      - run: {shell: true, command: "true"}
      - assert: {exit_code: 0}

Fixture suiteok.atago.yaml:

version: "1"
suite:
  name: inner
  setup:
    - run: {shell: true, command: "echo built"}
    - assert: {stdout: {contains: built}}
scenarios:
  - name: a
    steps:
      - run: {shell: true, command: "true"}
      - assert: {exit_code: 0}

When #

${atago} run suiteread.atago.yaml
${atago} run suiteok.atago.yaml

Then #

  • after ${atago} run suiteread.atago.yaml:
    • exit code is 2
    • stderr contains ATG2107, can never be fed here
  • after ${atago} run suiteok.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

atago self-hosting / exit_code in-set matcher #

Source: test/e2e/atago/exit_code_in.atago.yaml

Scenario: a listed exit code passes #

When #

exit 2

Then #

  • exit code is one of 0, 2

Scenario: an unlisted exit code fails and the output lists the set #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: exit 2 is not in the accepted set
    steps:
      - run:
          shell: true
          command: exit 2
      - assert:
          exit_code:
            in: [0, 1]

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains exit code in [0, 1]

Scenario: mixing not and in is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: both forms
    steps:
      - run:
          shell: true
          command: exit 0
      - assert:
          exit_code:
            not: 1
            in: [0, 2]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains exactly one of

Scenario: an empty in list is a load-time error #

Given #

  • Fixture file empty.atago.yaml is created.

Inputs #

Fixture empty.atago.yaml:

version: "1"
suite:
  name: empty
scenarios:
  - name: empty set
    steps:
      - run:
          shell: true
          command: exit 0
      - assert:
          exit_code:
            in: []

When #

${atago} run empty.atago.yaml

Then #

  • exit code is 2
  • stderr contains at least one accepted exit code

atago self-hosting / exit code semantics #

Source: test/e2e/atago/exit_codes.atago.yaml

Scenario: a clean exit is zero #

When #

exit 0

Then #

  • exit code is 0

Scenario: a general error is one #

When #

exit 1

Then #

  • exit code is 1

Scenario: a usage error is two #

When #

exit 2

Then #

  • exit code is 2

Scenario: an arbitrary code passes through unchanged #

When #

exit 42

Then #

  • exit code is 42

Scenario: the single-byte ceiling is 255 #

When #

exit 255

Then #

  • exit code is 255

Scenario: the not matcher excludes a specific code #

When #

exit 3

Then #

  • exit code is not 0

Scenario: the in matcher accepts any listed code #

When #

exit 2

Then #

  • exit code is one of 0, 1, 2

Scenario: an unlisted code fails the in matcher and names the set #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: five is not in the accepted set
    steps:
      - run: {shell: true, command: "exit 5"}
      - assert:
          exit_code:
            in: [0, 1, 2]

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains exit code in [0, 1, 2]

Scenario: SIGKILL is reported as 137 #

skipped on Windows

When #

kill -KILL $$

Then #

  • exit code is 137

Scenario: SIGTERM is reported as 143 #

skipped on Windows

When #

kill -TERM $$

Then #

  • exit code is 143

Scenario: SIGINT is reported as 130 #

skipped on Windows

When #

kill -INT $$

Then #

  • exit code is 130

Scenario: a signal exit composes with the in matcher alongside normal codes #

skipped on Windows

When #

kill -TERM $$

Then #

  • exit code is one of 0, 143

Scenario: a missing command is 127 under the shell #

skipped on Windows

When #

no_such_command_zzz

Then #

  • exit code is 127

Scenario: POSIX exit codes wrap modulo 256 #

skipped on Windows

When #

exit 257

Then #

  • exit code is 1

atago self-hosting / expected failures #

Source: test/e2e/atago/expect_fail.atago.yaml

Scenario: a known bug that still fails keeps the run green #

Given #

  • Fixture file inner_xfail.atago.yaml is created.

Inputs #

Fixture inner_xfail.atago.yaml:

version: "1"
suite:
  name: inner xfail
scenarios:
  - name: the bug is still there
    expect_fail:
      reason: "rounds half-down instead of half-even"
      issue: "https://example.test/issues/42"
    steps:
      - run:
          command: echo total=2.5
      - assert:
          stdout:
            contains: "total=2.50"

When #

${atago} run inner_xfail.atago.yaml

Then #

  • exit code is 0
  • stdout contains XFAIL:, rounds half-down instead of half-even, https://example.test/issues/42, 1 xfail

Scenario: a known bug that is fixed fails the run so it gets promoted #

Given #

  • Fixture file inner_xpass.atago.yaml is created.

Inputs #

Fixture inner_xpass.atago.yaml:

version: "1"
suite:
  name: inner xpass
scenarios:
  - name: the bug is fixed
    expect_fail:
      reason: "used to print nothing at all"
      issue: "https://example.test/issues/7"
    steps:
      - run:
          command: echo present
      - assert:
          stdout:
            contains: present

When #

${atago} run inner_xpass.atago.yaml

Then #

  • exit code is 1
  • stdout contains XPASS:, Move it into the suite, 1 xpass

Scenario: allow-xpass keeps the run green while the spec is promoted #

Given #

  • Fixture file inner_allow.atago.yaml is created.

Inputs #

Fixture inner_allow.atago.yaml:

version: "1"
suite:
  name: inner allow
scenarios:
  - name: the bug is fixed
    expect_fail:
      reason: "used to crash"
    steps:
      - run:
          command: echo ok
      - assert:
          stdout:
            contains: ok

When #

${atago} run --allow-xpass inner_allow.atago.yaml
${atago} run --allow-xpass --report junit inner_allow.atago.yaml

Then #

  • after ${atago} run --allow-xpass inner_allow.atago.yaml:
    • exit code is 0
    • stdout contains 1 xpass
  • after ${atago} run --allow-xpass --report junit inner_allow.atago.yaml:
    • exit code is 0
    • stdout contains failures="0", does not contain <failure

Scenario: fail-fast stops for an xpass and the ledger can rerun it #

Given #

  • Fixture file inner_ff.atago.yaml is created.

Inputs #

Fixture inner_ff.atago.yaml:

version: "1"
suite:
  name: inner fail fast
scenarios:
  - name: the bug is fixed
    expect_fail:
      reason: "used to print nothing at all"
    steps:
      - run:
          command: echo present
      - assert:
          stdout:
            contains: present
  - name: never reached
    steps:
      - run:
          command: echo later
      - assert:
          exit_code: 0

When #

${atago} run --fail-fast --parallel 1 --report json inner_ff.atago.yaml
${atago} run --rerun-failed --report json inner_ff.atago.yaml

Then #

  • after ${atago} run --fail-fast --parallel 1 --report json inner_ff.atago.yaml:
    • exit code is 1
    • stdout at $.suites[0].scenarios[0].status equals xpass; at $.suites[0].scenarios[1].status equals skipped
  • after ${atago} run --rerun-failed --report json inner_ff.atago.yaml:
    • exit code is 1
    • stdout at $.suites[0].scenarios has length 1; at $.suites[0].scenarios[0].status equals xpass

Scenario: an allowed xpass leaves nothing behind to rerun #

Given #

  • Fixture file inner_allow_ledger.atago.yaml is created.

Inputs #

Fixture inner_allow_ledger.atago.yaml:

version: "1"
suite:
  name: inner allow ledger
scenarios:
  - name: the bug is fixed
    expect_fail:
      reason: "used to crash"
    steps:
      - run:
          command: echo ok
      - assert:
          stdout:
            contains: ok

When #

${atago} run --allow-xpass inner_allow_ledger.atago.yaml
${atago} run --rerun-failed inner_allow_ledger.atago.yaml

Then #

  • after ${atago} run --allow-xpass inner_allow_ledger.atago.yaml:
    • exit code is 0
  • after ${atago} run --rerun-failed inner_allow_ledger.atago.yaml:
    • exit code is 0
    • stderr contains nothing to rerun

Scenario: an execution error is still an error, not an expected failure #

Given #

  • Fixture file inner_error.atago.yaml is created.

Inputs #

Fixture inner_error.atago.yaml:

version: "1"
suite:
  name: inner error
scenarios:
  - name: the spec itself cannot run
    expect_fail:
      reason: "the tool mishandles empty input"
    steps:
      - run:
          command: atago-no-such-binary-4f81

When #

${atago} run inner_error.atago.yaml

Then #

  • exit code is 4
  • stdout contains ERROR:, does not contain XFAIL:

Scenario: a skip gate still decides on its own #

Given #

  • Fixture file inner_skip.atago.yaml is created.
  • Environment variables are set: ATAGO_E2E_INNER_SKIP.

Inputs #

Fixture inner_skip.atago.yaml:

version: "1"
suite:
  name: inner skip
scenarios:
  - name: not applicable here
    expect_fail:
      reason: "only reproducible on plan9"
    skip:
      env: ATAGO_E2E_INNER_SKIP
    steps:
      - run:
          command: echo whatever
      - assert:
          stdout:
            contains: nope

When #

${atago} run inner_skip.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 skipped

Scenario: an expected failure is not retried into a flake #

Given #

  • Fixture file inner_retry.atago.yaml is created.
  • Environment variables are set: ATTEMPTS_FILE.
  • Environment variables are set: ATTEMPTS_FILE.

Inputs #

Fixture inner_retry.atago.yaml:

version: "1"
suite:
  name: inner retry
scenarios:
  - name: the bug is still there
    expect_fail:
      reason: "still broken"
    steps:
      - run:
          shell: true
          # A per-attempt side effect OUTSIDE the scenario workdir,
          # so the count survives the fresh workdir each attempt
          # gets. Asserting only on the report would pass an
          # implementation that ran three times and hid the word
          # "flaky".
          command: "echo x >> $${ATTEMPTS_FILE}; echo actual"
      - assert:
          stdout:
            contains: expected

When #

${atago} run --retry-failed 3 inner_retry.atago.yaml
${atago} run --repeat 3 inner_retry.atago.yaml

Then #

  • after ${atago} run --retry-failed 3 inner_retry.atago.yaml:
    • exit code is 0
    • stdout contains 1 xfail, does not contain flaky
    • file retry.count contains x exactly 1 time
  • after ${atago} run --repeat 3 inner_retry.atago.yaml:
    • exit code is 0
    • stdout contains 1 xfail, does not contain REPEAT:
    • file repeat.count contains x exactly 1 time

Scenario: tap marks both verdicts with a TODO directive #

Given #

  • Fixture file inner_tap.atago.yaml is created.

Inputs #

Fixture inner_tap.atago.yaml:

version: "1"
suite:
  name: inner tap
scenarios:
  - name: still broken
    expect_fail:
      reason: "known"
    steps:
      - run:
          command: echo a
      - assert:
          stdout:
            contains: b
  - name: now fixed
    expect_fail:
      reason: "known too"
    steps:
      - run:
          command: echo a
      - assert:
… (truncated, 2 more lines)

When #

${atago} run --report tap inner_tap.atago.yaml

Then #

  • exit code is 1
  • stdout contains not ok 1 - inner tap / still broken # TODO known, ok 2 - inner tap / now fixed # TODO known too

Scenario: junit routes an xfail to skipped and an xpass to failure #

Given #

  • Fixture file inner_junit.atago.yaml is created.

Inputs #

Fixture inner_junit.atago.yaml:

version: "1"
suite:
  name: inner junit
scenarios:
  - name: still broken
    expect_fail:
      reason: "known"
    steps:
      - run:
          command: echo a
      - assert:
          stdout:
            contains: b
  - name: now fixed
    expect_fail:
      reason: "known too"
    steps:
      - run:
          command: echo a
      - assert:
… (truncated, 2 more lines)

When #

${atago} run --report junit inner_junit.atago.yaml

Then #

  • exit code is 1
  • stdout contains skipped="1", failures="1", <skipped message="xfail: known">, <failure message="xpass:

Scenario: explain and doc show which scenarios document a known bug #

Given #

  • Fixture file shown.atago.yaml is created.

Inputs #

Fixture shown.atago.yaml:

version: "1"
suite:
  name: shown
scenarios:
  - name: known bug
    expect_fail:
      reason: "wrong rounding"
      issue: "https://example.test/9"
    steps:
      - run:
          command: echo hi

When #

${atago} explain shown.atago.yaml
${atago} doc shown.atago.yaml

Then #

  • after ${atago} explain shown.atago.yaml:
    • exit code is 0
    • stdout contains [expect_fail: wrong rounding], https://example.test/9
  • after ${atago} doc shown.atago.yaml:
    • exit code is 0
    • stdout contains expected to FAIL (known bug): wrong rounding

Scenario: update-snapshots keeps the golden of an expected failure #

Given #

  • Fixture file desired.snap is created.
  • Fixture file inner_snap.atago.yaml is created.

Inputs #

Fixture desired.snap:

the CORRECT output once fixed

Fixture inner_snap.atago.yaml:

version: "1"
suite:
  name: inner xfail snap
scenarios:
  - name: the output is still wrong
    expect_fail:
      reason: "prints the pre-fix output"
    steps:
      - run:
          shell: true
          command: echo current buggy output
      - assert:
          stdout:
            snapshot: desired.snap

When #

${atago} run --update-snapshots inner_snap.atago.yaml
${atago} snapshot update inner_snap.atago.yaml

Then #

  • after ${atago} run --update-snapshots inner_snap.atago.yaml:
    • exit code is 0
    • stdout contains XFAIL:, 1 xfail, does not contain snapshot updated
    • file desired.snap contains the CORRECT output once fixed
  • after ${atago} snapshot update inner_snap.atago.yaml:
    • exit code is 0
    • file desired.snap contains the CORRECT output once fixed

Scenario: update-snapshots still reports the day the documented bug is fixed #

Given #

  • Fixture file fixed.snap is created.
  • Fixture file inner_fixed.atago.yaml is created.

Inputs #

Fixture fixed.snap:

fixed output

Fixture inner_fixed.atago.yaml:

version: "1"
suite:
  name: inner xpass snap
scenarios:
  - name: the output now matches the golden
    expect_fail:
      reason: "prints the pre-fix output"
    steps:
      - run:
          shell: true
          command: echo fixed output
      - assert:
          stdout:
            snapshot: fixed.snap

When #

${atago} run --update-snapshots inner_fixed.atago.yaml

Then #

  • exit code is 1
  • stdout contains XPASS:, 1 xpass

Scenario: an ordinary scenario beside an expected failure is still re-recorded #

Given #

  • Fixture file ordinary.snap is created.
  • Fixture file frozen.snap is created.
  • Fixture file inner_mixed.atago.yaml is created.

Inputs #

Fixture ordinary.snap:

stale

Fixture frozen.snap:

desired

Fixture inner_mixed.atago.yaml:

version: "1"
suite:
  name: inner mixed
scenarios:
  - name: ordinary
    steps:
      - run:
          shell: true
          command: echo fresh
      - assert:
          stdout:
            snapshot: ordinary.snap
  - name: documents a bug
    expect_fail:
      reason: "still broken"
    steps:
      - run:
          shell: true
          command: echo buggy
      - assert:
… (truncated, 2 more lines)

When #

${atago} run --update-snapshots inner_mixed.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 snapshot updated
  • file ordinary.snap contains fresh
  • file frozen.snap contains desired

Scenario: an expected failure without a reason is a load error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: unexplained
    expect_fail:
      issue: "https://example.test/1"
    steps:
      - run:
          command: echo hi

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains expect_fail.reason is required

atago self-hosting / explain #

Source: test/e2e/atago/explain.atago.yaml

Scenario: explain summarizes a spec without running it #

Given #

  • Fixture file target.atago.yaml is created.

Inputs #

Fixture target.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: list as json
    steps:
      - run:
          command: echo "[]"
      - assert:
          exit_code: 0
          stdout:
            contains: "["

When #

${atago} explain target.atago.yaml

Then #

  • exit code is 0
  • stdout contains Suite: sample, Scenario: list as json, Commands:, Network policy:

Scenario: explain describes every matcher of a composed stream assertion #

Given #

  • Fixture file composed.atago.yaml is created.
  • Fixture file broken.atago.yaml is created.

Inputs #

Fixture composed.atago.yaml:

version: "1"
suite:
  name: composed
scenarios:
  - name: guards both directions
    steps:
      - run:
          command: echo hello
      - assert:
          stdout:
            contains: hello
            not_contains: goodbye
            matches: "^hel"

Fixture broken.atago.yaml:

version: "1"
suite:
  name: composed
scenarios:
  - name: guards both directions
    steps:
      - run:
          shell: true
          command: printf 'hello goodbye\n'
      - assert:
          stdout:
            contains: hello
            not_contains: goodbye

When #

${atago} explain composed.atago.yaml
${atago} run broken.atago.yaml

Then #

  • after ${atago} explain composed.atago.yaml:
    • exit code is 0
    • stdout contains stdout contains "hello", does not contain "goodbye", matches /^hel/
  • after ${atago} run broken.atago.yaml:
    • exit code is 1
    • stdout contains does not contain

Scenario: explain names the line a line-scoped matcher inspects #

Given #

  • Fixture file lined.atago.yaml is created.

Inputs #

Fixture lined.atago.yaml:

version: "1"
suite:
  name: lined
scenarios:
  - name: second line only
    steps:
      - run:
          shell: true
          command: printf 'one\ntwo\n'
      - assert:
          stdout:
            line: 2
            equals: two

When #

${atago} explain lined.atago.yaml

Then #

  • exit code is 0
  • stdout contains stdout line 2 equals exact text

Scenario: explain describes file not_contains and executable matchers #

Given #

  • Fixture file filematch.atago.yaml is created.

Inputs #

Fixture filematch.atago.yaml:

version: "1"
suite:
  name: filematch
scenarios:
  - name: no secret leaked
    steps:
      - fixture:
          file: out.txt
          content: "public\n"
      - run:
          command: echo hi
      - assert:
          file:
            path: out.txt
            not_contains: "secret-token"
  - name: installer stays executable
    steps:
      - fixture:
          file: install.sh
          content: "#!/bin/sh\n"
… (truncated, 7 more lines)

When #

${atago} explain filematch.atago.yaml

Then #

  • exit code is 0
  • stdout contains file "out.txt" does not contain "secret-token", file "install.sh" is executable

Scenario: explain names ssh and remote-database egress #

Given #

  • Fixture file remote.atago.yaml is created.

Inputs #

Fixture remote.atago.yaml:

version: "1"
suite:
  name: remote
runners:
  # insecure_host_key is an explicit opt-out of a verification the
  # loader otherwise refuses to let a spec skip, so the summary has
  # to say which way the decision went.
  box: {type: ssh, host: shell.example, user: deploy, password: x, insecure_host_key: true}
  # Single-quoted: a double-quoted YAML scalar processes escapes, and
  # ${workdir} on Windows expands to a backslash path whose \A and \T
  # are not valid ones. explain never opens the database, so a plain
  # relative path says as much as an absolute one.
  pg: {type: db, dsn: 'postgres://u:p@db.example:5432/app'}
  local: {type: db, dsn: 'sqlite:a.db'}
scenarios:
  - name: reaches two hosts and one file
    steps:
      - run: {runner: box, command: uptime}
      - query: {runner
… (truncated)

When #

${atago} explain remote.atago.yaml
${atago} manifest remote.atago.yaml
${atago} doc remote.atago.yaml

Then #

  • after ${atago} explain remote.atago.yaml:
    • exit code is 0
    • stdout contains network access (ssh box): uptime, ssh host key verification disabled (runner "box"), network access: SQL query via pg, uptime (ssh box), does not contain network access: SQL query via local
  • after ${atago} manifest remote.atago.yaml:
    • exit code is 0
    • stdout at $.specs[0].scenarios[0].security[0] equals network access (ssh box): uptime; at $.specs[0].scenarios[0].steps[0].action equals run via ssh box: uptime
  • after ${atago} doc remote.atago.yaml:
    • exit code is 0
    • stdout contains # ssh box: uptime

Scenario: explain names pty and teardown egress #

Given #

  • Fixture file pty_teardown.atago.yaml is created.

Inputs #

Fixture pty_teardown.atago.yaml:

version: "1"
suite:
  name: pty-teardown
scenarios:
  - name: interactive deploy with cleanup
    steps:
      - pty:
          command: ssh deploy@backend.example
          session:
            - expect: "password:"
            - send: "$${env:DEPLOY_PASSWORD}\n"
            - exec: {command: "curl https://hook.example/fire"}
    teardown:
      - run:
          command: curl https://api.example/cleanup
          shell: true

When #

${atago} explain pty_teardown.atago.yaml
${atago} manifest pty_teardown.atago.yaml

Then #

  • after ${atago} explain pty_teardown.atago.yaml:
    • exit code is 0
    • stdout contains network access: ssh deploy@backend.example, host environment read: $${env:DEPLOY_PASSWORD}, network access (pty exec): curl https://hook.example/fire, shell execution enabled: curl https://api.example/cleanup, network access: curl https://api.example/cleanup
  • after ${atago} manifest pty_teardown.atago.yaml:
    • exit code is 0
    • stdout at $.specs[0].scenarios[0].security[0] equals network access: ssh deploy@backend.example

Scenario: explain names suite lifecycle egress #

Given #

  • Fixture file suite_egress.atago.yaml is created.

Inputs #

Fixture suite_egress.atago.yaml:

version: "1"
suite:
  name: suite-egress
  setup:
    - run:
        command: curl https://seed.example/data
        shell: true
    - service:
        name: relay
        command: ssh -N -L 8080:internal.example:80 jump.example
  teardown:
    - run:
        command: curl https://api.example/purge
        shell: true
scenarios:
  - name: quiet scenario
    steps:
      - run:
          command: echo hi

When #

${atago} explain suite_egress.atago.yaml
${atago} manifest suite_egress.atago.yaml

Then #

  • after ${atago} explain suite_egress.atago.yaml:
    • exit code is 0
    • stdout contains Suite security notes:, shell execution enabled: curl https://seed.example/data, network access (service relay): ssh -N -L 8080:internal.example:80 jump.example, network access: curl https://api.example/purge
  • after ${atago} manifest suite_egress.atago.yaml:
    • exit code is 0
    • stdout at $.specs[0].suite_security[0] equals shell execution enabled: curl https://seed.example/data

Scenario: explain names environment reads in fixtures asserts and cdp actions #

Given #

  • Fixture file env_reads.atago.yaml is created.

Inputs #

Fixture env_reads.atago.yaml:

version: "1"
suite:
  name: env-reads
runners:
  web: {type: browser}
scenarios:
  - name: reads the environment from quiet corners
    steps:
      - fixture:
          file: seed.txt
          content: "token=$${env:SEED_TOKEN}"
      - run:
          command: echo ready
      - assert:
          stdout:
            contains: "$${env:EXPECTED_GREETING}"
      - cdp:
          runner: web
          actions:
            - navigate: "https://host.example/?key=$${env:CDP_KEY}"

When #

${atago} explain env_reads.atago.yaml

Then #

  • exit code is 0
  • stdout contains host environment read: $${env:SEED_TOKEN}, host environment read: $${env:EXPECTED_GREETING}, host environment read: $${env:CDP_KEY}

Scenario: explain lists pdf and teardown outputs under generates #

Given #

  • Fixture file gen.atago.yaml is created.

Inputs #

Fixture gen.atago.yaml:

version: "1"
suite:
  name: gen
scenarios:
  - name: renders a report
    steps:
      - run:
          command: mytool render
      - assert:
          pdf:
            path: report.pdf
            min_pages: 1
    teardown:
      - run:
          command: mytool audit
          stdout_to: logs/audit.log

When #

${atago} explain gen.atago.yaml
${atago} manifest gen.atago.yaml

Then #

  • after ${atago} explain gen.atago.yaml:
    • exit code is 0
    • stdout contains Generates:, report.pdf, logs/audit.log
  • after ${atago} manifest gen.atago.yaml:
    • exit code is 0
    • stdout at $.specs[0].scenarios[0].generates[0] equals report.pdf; at $.specs[0].scenarios[0].generates[1] equals logs/audit.log

Scenario: explain names an http runner and describes a retry #

Given #

  • Fixture file httpretry.atago.yaml is created.

Inputs #

Fixture httpretry.atago.yaml:

version: "1"
suite:
  name: httpretry
runners:
  internal: {type: http, base_url: "http://127.0.0.1:8080"}
  billing: {type: http, base_url: "https://billing.example.com"}
scenarios:
  - name: talks to two hosts and polls one
    steps:
      - http: {runner: internal, method: GET, path: /health}
      - http:
          runner: billing
          method: POST
          path: /charge
          retry: {times: 3, interval: 200ms, until: {status: 200}}
      - assert: {status: 200}

When #

${atago} explain httpretry.atago.yaml
${atago} doc httpretry.atago.yaml
${atago} manifest httpretry.atago.yaml

Then #

  • after ${atago} explain httpretry.atago.yaml:
    • exit code is 0
    • stdout contains HTTP GET /health via internal, HTTP POST /charge via billing, network access: HTTP request via internal, network access: HTTP request via billing, retried up to 3 times every 200ms until HTTP status is 200
  • after ${atago} doc httpretry.atago.yaml:
    • exit code is 0
    • stdout contains # HTTP POST /charge via billing, The step is retried up to 3 times every 200ms until HTTP status is `200`.
  • after ${atago} manifest httpretry.atago.yaml:
    • exit code is 0
    • stdout at $.specs[0].scenarios[0].steps[1].action equals HTTP POST /charge via billing; at $.specs[0].scenarios[0].steps[1].retry.until equals status

Scenario: explain names a runner definition’s host environment reads #

Given #

  • Fixture file runnerenv.atago.yaml is created.

Inputs #

Fixture runnerenv.atago.yaml:

version: "1"
suite:
  name: runnerenv
runners:
  pg: {type: db, dsn: 'postgres://u:$${env:DB_PASSWORD}@db.example:5432/app'}
scenarios:
  - name: queries through the runner
    steps:
      - query: {runner: pg, sql: "SELECT 1"}

When #

${atago} explain runnerenv.atago.yaml
${atago} manifest runnerenv.atago.yaml

Then #

  • after ${atago} explain runnerenv.atago.yaml:
    • exit code is 0
    • stdout contains host environment read: $${env:DB_PASSWORD}, Variables used: env:DB_PASSWORD
  • after ${atago} manifest runnerenv.atago.yaml:
    • exit code is 0
    • stdout at $.specs[0].scenarios[0].variables[0] equals env:DB_PASSWORD

Scenario: explain names env and command gates #

Given #

  • Fixture file gates.atago.yaml is created.

Inputs #

Fixture gates.atago.yaml:

version: "1"
suite:
  name: gates
defaults:
  scenario:
    only:
      command: "jq --version"
scenarios:
  - name: gated by the suite default
    steps:
      - run: {shell: true, command: "echo hi"}
  - name: gated on an environment variable
    skip: {env: CI}
    steps:
      - run: {shell: true, command: "echo hi"}

When #

${atago} explain gates.atago.yaml

Then #

  • exit code is 0
  • stdout contains [only command="jq --version"], [skip env=CI]

atago self-hosting / file equals and equals_file byte-equality (#155) #

Source: test/e2e/atago/file_equals.atago.yaml

Scenario: equals_file passes for two byte-identical files #

Given #

  • Fixture file in.hex is created.
  • Fixture file out.hex is created.

Inputs #

Fixture in.hex:

DEADBEEF

Fixture out.hex:

DEADBEEF

Then #

  • file out.hex is byte-identical to in.hex

Scenario: equals matches an inline literal byte-for-byte #

Given #

  • Fixture file token.txt is created.

Inputs #

Fixture token.txt:

opaque-value-42

Then #

  • file token.txt equals exact bytes

Scenario: equals_file fails the inner spec when the two files differ by one byte #

Given #

  • Fixture file neq.atago.yaml is created.

Inputs #

Fixture neq.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: two files differ by one byte
    steps:
      - fixture:
          file: a.hex
          content: "DEADBEEF"
      - fixture:
          file: b.hex
          content: "DEADBEEE"
      - assert:
          file:
            path: a.hex
            equals_file: b.hex

When #

${atago} run neq.atago.yaml

Then #

  • exit code is 1
  • stdout contains not byte-identical

Scenario: equals_file is byte-exact — a CRLF vs LF difference fails #

Given #

  • Fixture file crlf.atago.yaml is created.

Inputs #

Fixture crlf.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: LF and CRLF files are not byte-identical
    steps:
      - fixture:
          file: lf.txt
          base64: "bGluZQo="
      - fixture:
          file: crlf.txt
          base64: "bGluZQ0K"
      - assert:
          file:
            path: lf.txt
            equals_file: crlf.txt

When #

${atago} run crlf.atago.yaml

Then #

  • exit code is 1
  • stdout contains not byte-identical

Scenario: a file assertion is not satisfied by a directory #

Given #

  • Fixture file mistaken.atago.yaml is created.

Inputs #

Fixture mistaken.atago.yaml:

version: "1"
suite: {name: mistaken}
scenarios:
  - name: out is a directory, not the expected file
    steps:
      - run: {shell: true, command: "mkdir out"}
      - assert:
          file:
            path: out
            exists: true

When #

${atago} run mistaken.atago.yaml

Then #

  • exit code is 1
  • stdout contains exists but is a directory, use a dir: assertion

Scenario: an executable assertion is not satisfied by a directory #

skipped on Windows

Given #

  • Fixture file execdir.atago.yaml is created.

Inputs #

Fixture execdir.atago.yaml:

version: "1"
suite: {name: execdir}
scenarios:
  - name: a directory is not an executable
    steps:
      - run: {shell: true, command: "mkdir bin"}
      - assert:
          file:
            path: bin
            executable: true

When #

${atago} run execdir.atago.yaml

Then #

  • exit code is 1
  • stdout contains is a directory, not an executable file

Scenario: a real file still satisfies both matchers #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite: {name: ok}
scenarios:
  - name: a real executable file
    steps:
      - fixture:
          file: tool.bat
          content: "@echo hi\n"
          mode: "0755"
      - fixture:
          file: notes.txt
          content: "plain\n"
          mode: "0644"
      - run: {command: echo hi}
      - assert:
          file: {path: tool.bat, exists: true}
      - assert:
          file: {path: tool.bat, executable: true}
      # The negative half, also portable: no execute bit on POSIX,
      # and .txt is not in PATHEXT on Windows.
… (truncated, 2 more lines)

When #

${atago} run ok.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: an executable failure on Windows names PATHEXT rather than a mode #

only on Windows

Given #

  • Fixture file noext.atago.yaml is created.

Inputs #

Fixture noext.atago.yaml:

version: "1"
suite: {name: noext}
scenarios:
  - name: a POSIX script is not runnable by name on Windows
    steps:
      - fixture:
          file: tool.sh
          content: "echo hi\n"
          mode: "0755"
      - assert:
          file: {path: tool.sh, executable: true}

When #

${atago} run noext.atago.yaml

Then #

  • exit code is 1
  • stdout contains PATHEXT, extension ".sh", does not contain (mode

Scenario: an invisible difference is quoted in the failure output #

Given #

  • Fixture file crlf.atago.yaml is created.

Inputs #

Fixture crlf.atago.yaml:

version: "1"
suite: {name: crlf}
scenarios:
  - name: the file ends with CRLF, the expectation with LF
    steps:
      - fixture:
          file: a.txt
          base64: c2hhcmVkDQo=
      - run: {command: echo hi}
      - assert:
          file:
            path: a.txt
            equals: "shared\n"

When #

${atago} run crlf.atago.yaml

Then #

  • exit code is 1
  • stdout contains "shared\n", "shared\r\n"

Scenario: a trailing space difference is quoted too #

Given #

  • Fixture file spaces.atago.yaml is created.

Inputs #

Fixture spaces.atago.yaml:

version: "1"
suite: {name: spaces}
scenarios:
  - name: the output carries trailing spaces
    steps:
      - run:
          shell: true
          command: "printf 'value  '"
      - assert:
          stdout:
            equals: "value"

When #

${atago} run spaces.atago.yaml

Then #

  • exit code is 1
  • stdout contains "value "

Scenario: an ordinary difference keeps its plain form #

Given #

  • Fixture file plain.atago.yaml is created.

Inputs #

Fixture plain.atago.yaml:

version: "1"
suite: {name: plain}
scenarios:
  - name: the texts really are different
    steps:
      - run:
          shell: true
          command: "printf 'actual text'"
      - assert:
          stdout:
            equals: "expected text"

When #

${atago} run plain.atago.yaml

Then #

  • exit code is 1
  • stdout contains expected text, actual text
  • stdout does not contain "actual text"

atago self-hosting / fixture from (copy committed testdata) #

Source: test/e2e/atago/fixture_from.atago.yaml

Scenario: a committed binary blob is copied verbatim into the workdir #

Given #

  • Fixture file copied.bin is created.

When #

wc -c < copied.bin

Then #

  • exit code is 0
  • stdout contains 21
  • file copied.bin contains binary-marker

Scenario: copying from a missing source errors the scenario #

Given #

  • Fixture file copied.bin is created.

Inputs #

Fixture copied.bin:

version: "1"
suite:
  name: inner
scenarios:
  - name: missing source
    steps:
      - fixture:
          file: x.bin
          from: testdata/does-not-exist.bin
      - run:
          command: "true"

When #

${atago} run copied.bin

Then #

  • exit code is 4
  • stdout contains copy from

atago self-hosting / fixture symlink+mode+mtime, file executable, env skip #

Source: test/e2e/atago/fixture_modes.atago.yaml

Given #

  • Fixture file target.txt is created.
  • Fixture file alias.txt is created.

Inputs #

Fixture target.txt:

payload

When #

cat alias.txt

Then #

  • stdout equals an exact value

Scenario: fixture.mode sets permissions and file.executable reads them #

Given #

  • Fixture file run.sh is created.
  • Fixture file data.txt is created.

Inputs #

Fixture run.sh:

echo hi

Fixture data.txt:

plain

Then #

  • file run.sh is executable
  • file data.txt is not executable

Scenario: fixture.mtime pins the modification time #

Given #

  • Fixture file stamped.txt is created.

Inputs #

Fixture stamped.txt:

x

When #

date -u -r stamped.txt +%Y

Then #

  • stdout contains 2021

Scenario: only.env skips when the variable is unset #

only when env ATAGO_DEFINITELY_UNSET is set

When #

false

Scenario: skip.env runs when the variable is unset #

skipped when env ATAGO_DEFINITELY_UNSET is set

When #

true

Then #

  • exit code is 0

atago self-hosting / flaky tooling (–repeat, –retry-failed) #

Source: test/e2e/atago/flaky.atago.yaml

Scenario: retry-failed recovers a flaky scenario and reports it loudly #

skipped on Windows

Given #

  • Fixture file flaky.atago.yaml is created.

Inputs #

Fixture flaky.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: flaky once
    steps:
      - run:
          shell: true
          command: "if [ -f '${workdir}/seen.txt' ]; then echo recovered; else touch '${workdir}/seen.txt'; exit 1; fi"
      - assert:
          exit_code: 0

When #

${atago} run flaky.atago.yaml
rm -f seen.txt
${atago} run --retry-failed 1 flaky.atago.yaml
rm -f seen.txt
${atago} run --retry-failed 1 --allow-flaky flaky.atago.yaml
rm -f seen.txt
${atago} run --retry-failed 1 --report json flaky.atago.yaml

Then #

  • after ${atago} run flaky.atago.yaml:
    • exit code is 1
  • after ${atago} run --retry-failed 1 flaky.atago.yaml:
    • exit code is 1
    • stdout contains FLAKY:, passed after 2 attempts, 1 flaky
  • after ${atago} run --retry-failed 1 --allow-flaky flaky.atago.yaml:
    • exit code is 0
    • stdout contains FLAKY:, passed after 2 attempts, 1 flaky
  • after ${atago} run --retry-failed 1 --report json flaky.atago.yaml:
    • exit code is 1
    • stdout contains "status": "flaky", "attempts": 2

Scenario: repeat surfaces flakiness that a single run would miss #

skipped on Windows

Given #

  • Fixture file green.atago.yaml is created.
  • Fixture file flaky.atago.yaml is created.
  • Fixture file broken.atago.yaml is created.

Inputs #

Fixture green.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: steady
    steps:
      - run: {shell: true, command: echo ok}
      - assert:
          exit_code: 0

Fixture flaky.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: flaky once
    steps:
      - run:
          shell: true
          command: "if [ -f '${workdir}/seen.txt' ]; then echo recovered; else touch '${workdir}/seen.txt'; exit 1; fi"
      - assert:
          exit_code: 0

Fixture broken.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: always fails
    steps:
      - run: {shell: true, command: exit 1}
      - assert:
          exit_code: 0

When #

${atago} run --repeat 3 green.atago.yaml
${atago} run --repeat 3 flaky.atago.yaml
rm -f seen.txt
${atago} run --repeat 3 --allow-flaky flaky.atago.yaml
${atago} run --repeat 3 broken.atago.yaml

Then #

  • after ${atago} run --repeat 3 green.atago.yaml:
    • exit code is 0
    • stdout contains steady: 3/3 passed
  • after ${atago} run --repeat 3 flaky.atago.yaml:
    • exit code is 1
    • stdout contains flaky once: 2/3 passed, 1 flaky
  • after ${atago} run --repeat 3 --allow-flaky flaky.atago.yaml:
    • exit code is 0
    • stdout contains flaky once: 2/3 passed, 1 flaky
  • after ${atago} run --repeat 3 broken.atago.yaml:
    • exit code is 1
    • stdout contains always fails: 0/3 passed, 1 failed

Scenario: a gated-out scenario reports no repeat rate #

Given #

  • Fixture file gated.atago.yaml is created.

Inputs #

Fixture gated.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: needs a variable nobody set
    only: {env: ATAGO_E2E_NEVER_SET}
    steps:
      - run: {shell: true, command: exit 1}
      - assert:
          exit_code: 0

When #

${atago} run --repeat 3 gated.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 skipped, does not contain 0/3 passed

Scenario: repeat and retry-failed are mutually exclusive #

Given #

  • Fixture file any.atago.yaml is created.

Inputs #

Fixture any.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: s
    steps:
      - run: {command: echo hi}

When #

${atago} run --repeat 2 --retry-failed 1 any.atago.yaml

Then #

  • exit code is 3
  • stderr contains mutually exclusive

atago self-hosting / grpc runner #

Source: test/e2e/atago/grpc.atago.yaml

Scenario: a grpc runner without a target fails validation (exit 2) #

Given #

  • Fixture file badgrpc.atago.yaml is created.

Inputs #

Fixture badgrpc.atago.yaml:

version: "1"
suite:
  name: grpc
runners:
  g:
    type: grpc
scenarios:
  - name: call
    steps:
      - grpc:
          runner: g
          method: pkg.Service/Method

When #

${atago} run badgrpc.atago.yaml

Then #

  • exit code is 2
  • stderr contains requires a target

Scenario: a grpc step naming an undeclared runner fails validation (exit 2) #

Given #

  • Fixture file norunner.atago.yaml is created.

Inputs #

Fixture norunner.atago.yaml:

version: "1"
suite:
  name: grpc
scenarios:
  - name: call
    steps:
      - grpc:
          runner: missing
          method: pkg.Service/Method

When #

${atago} run norunner.atago.yaml

Then #

  • exit code is 2
  • stderr contains is not declared

Scenario: an unreachable server is a connection failure, not a reflection question #

Given #

  • Fixture file dead.atago.yaml is created.

Inputs #

Fixture dead.atago.yaml:

version: "1"
suite:
  name: inner
runners:
  rpc: {type: grpc, target: "127.0.0.1:1", timeout: 5s}
scenarios:
  - name: call a server that is not there
    steps:
      - grpc:
          runner: rpc
          method: pkg.Service/Method

When #

${atago} run dead.atago.yaml

Then #

  • exit code is 4
  • stdout contains the server could not be reached, does not contain server reflection enabled

atago self-hosting / hermetic environment (clear_env + pass_env) #

Source: test/e2e/atago/hermetic_env.atago.yaml

Scenario: clear_env drops inherited host variables #

skipped on Windows

Given #

  • The command runs with a cleared environment.

When #

env
env

Then #

  • after env:
    • exit code is 0
    • stdout contains ATAGO_HERMETIC_CANARY
  • after env:
    • exit code is 0
    • stdout contains ATAGO_HERMETIC_CANARY=leaked-from-scenario
    • stdout does not contain PATH=/

Scenario: pass_env re-admits an allowlist of host variables #

skipped on Windows

Given #

  • The command runs with a cleared environment (passing through: PATH).

When #

env

Then #

  • exit code is 0
  • stdout contains PATH=
  • stdout does not contain HOME=

Scenario: explicit env wins over a passed-through host variable #

skipped on Windows

Given #

  • Environment variables are set: HOME.
  • The command runs with a cleared environment (passing through: HOME).

When #

printf '%s\n' "$HOME"

Then #

  • exit code is 0
  • stdout equals an exact value

Scenario: pass_env without clear_env is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: forgot clear_env
    steps:
      - run:
          command: env
          pass_env: [PATH]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains pass_env requires clear_env: true, steps[0].run

Scenario: unset host variables in pass_env are skipped, not an error #

skipped on Windows

Given #

  • The command runs with a cleared environment (passing through: PATH, ATAGO_SURELY_UNSET_VAR_2026).

When #

env

Then #

  • exit code is 0
  • stdout contains PATH=
  • stdout does not contain ATAGO_SURELY_UNSET_VAR_2026

atago self-hosting / http runner #

Source: test/e2e/atago/http.atago.yaml

Scenario: a denied host is a security policy violation (exit 6) #

Given #

  • Fixture file denied.atago.yaml is created.

Inputs #

Fixture denied.atago.yaml:

version: "1"
suite:
  name: api
permissions:
  network:
    allow:
      - allowed.example
runners:
  api:
    type: http
    base_url: http://127.0.0.1:1
scenarios:
  - name: request to a non-allowlisted host
    steps:
      - http:
          runner: api
          method: GET
          path: /ping

When #

${atago} run denied.atago.yaml

Then #

  • exit code is 6
  • stdout contains network policy denies

Scenario: a db runner dialing a denied host is the same violation (exit 6) #

Given #

  • Fixture file deniedb.atago.yaml is created.

Inputs #

Fixture deniedb.atago.yaml:

version: "1"
suite:
  name: db
permissions:
  network:
    allow:
      - allowed.example
runners:
  store:
    type: db
    dsn: "postgres://u:p@denied.example:5432/app?sslmode=disable"
scenarios:
  - name: query against a non-allowlisted host
    steps:
      - query:
          runner: store
          sql: "SELECT 1"

When #

${atago} run deniedb.atago.yaml

Then #

  • exit code is 6
  • stdout contains network policy denies host "denied.example"

Scenario: a file-backed db runner is not egress #

Given #

  • Fixture file localdb.atago.yaml is created.

Inputs #

Fixture localdb.atago.yaml:

version: "1"
suite:
  name: db
permissions:
  network:
    allow:
      - allowed.example
runners:
  store:
    type: db
    dsn: sqlite:${workdir}/app.db
scenarios:
  - name: a local database still runs
    steps:
      - query:
          runner: store
          sql: "CREATE TABLE t (a INTEGER)"

When #

${atago} run localdb.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: an http step with an undeclared runner fails validation (exit 2) #

Given #

  • Fixture file norunner.atago.yaml is created.

Inputs #

Fixture norunner.atago.yaml:

version: "1"
suite:
  name: api
scenarios:
  - name: references a runner that is not declared
    steps:
      - http:
          runner: missing
          method: GET
          path: /ping

When #

${atago} run norunner.atago.yaml

Then #

  • exit code is 2
  • stderr contains is not declared

atago self-hosting / image #

Source: test/e2e/atago/image.atago.yaml

Scenario: format, dimension and alpha assertions pass on a PNG #

Given #

  • Fixture file img.atago.yaml is created.

Inputs #

Fixture img.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: a red 2x2 png
    steps:
      - fixture:
          file: red.png
          base64: iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAFElEQVR4nGL5zwACTGCSARAAAP//DUcBBkxXt5sAAAAASUVORK5CYII=
      - assert:
          image:
            path: red.png
            format: png
            width: 2
            height: 2
            min_width: 1
            max_width: 100
            # This is an opaque truecolor (color-type 2) PNG with no
            # alpha channel, so alpha is false (issue #13).
            alpha: false

When #

${atago} run img.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: a pixel comparison against an identical baseline passes #

Given #

  • Fixture file baseline.png is created.
  • Fixture file sim.atago.yaml is created.

Inputs #

Fixture sim.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: output matches the committed baseline
    steps:
      - fixture:
          file: out.png
          base64: iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAGUlEQVR4nGJJSUlhwAaYsIoOWglAAAAA///xaAE/jf0lQAAAAABJRU5ErkJggg==
      - assert:
          image:
            path: out.png
            similar_to: ${workdir}/baseline.png

When #

${atago} run sim.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: a wrong dimension assertion fails with a clear diff #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: width does not match
    steps:
      - fixture:
          file: red.png
          base64: iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAFElEQVR4nGL5zwACTGCSARAAAP//DUcBBkxXt5sAAAAASUVORK5CYII=
      - assert:
          image:
            path: red.png
            width: 999

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 1
  • stdout contains FAILED

Scenario: a failing similar_to writes visual diff artifacts #

Given #

  • Fixture file baseline.png is created.
  • Fixture file diff.atago.yaml is created.

Inputs #

Fixture diff.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: output differs from the baseline
    steps:
      - fixture:
          file: out.png
          base64: iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAGElEQVR4nGJhYPjPAANMDEgANwcQAAD//zJvAQo6fkx6AAAAAElFTkSuQmCC
      - assert:
          image:
            path: out.png
            similar_to: ${workdir}/baseline.png

When #

${atago} run --report json --artifacts-dir arts diff.atago.yaml
ls arts/*/*/
cat arts/*/*/*image.metadata.json

Then #

  • after ${atago} run --report json --artifacts-dir arts diff.atago.yaml:
    • exit code is 1
    • stdout contains "artifacts"
    • dir arts exists
  • after ls arts/*/*/:
    • exit code is 0
    • stdout contains image.actual.png, image.baseline.png, image.diff.png, image.metadata.json
  • after cat arts/*/*/*image.metadata.json:
    • exit code is 0
    • stdout at $.diff_generated equals true

Scenario: a similar_to baseline resolves in the scenario workdir #

Given #

  • Fixture file roundtrip.atago.yaml is created.

Inputs #

Fixture roundtrip.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: a copy of the output matches it pixel for pixel
    steps:
      - fixture:
          file: source.png
          base64: iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAGUlEQVR4nGJJSUlhwAaYsIoOWglAAAAA///xaAE/jf0lQAAAAABJRU5ErkJggg==
      - run:
          shell: true
          command: cp source.png restored.png
      - assert:
          exit_code: 0
          image:
            path: restored.png
            similar_to: source.png

When #

${atago} run roundtrip.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: a workdir baseline still fails when the images differ #

Given #

  • Fixture file differ.atago.yaml is created.

Inputs #

Fixture differ.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: a different image is not the baseline
    steps:
      - fixture:
          file: source.png
          base64: iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFUlEQVR4nGL5z4AATAxEcQABAAD//zRtAQqfxpGAAAAAAElFTkSuQmCC
      - fixture:
          file: other.png
          base64: iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAGElEQVR4nGJhYPjPAANMDEgANwcQAAD//zJvAQo6fkx6AAAAAElFTkSuQmCC
      - run:
          command: echo compared
      - assert:
          image:
            path: other.png
            similar_to: source.png

When #

${atago} run differ.atago.yaml

Then #

  • exit code is 1
  • stdout contains FAILED

Scenario: a baseline that exists nowhere names both places it was looked for #

Given #

  • Fixture file missing.atago.yaml is created.

Inputs #

Fixture missing.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: the baseline was never produced
    steps:
      - fixture:
          file: out.png
          base64: iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFUlEQVR4nGL5z4AATAxEcQABAAD//zRtAQqfxpGAAAAAAElFTkSuQmCC
      - run:
          command: echo compared
      - assert:
          image:
            path: out.png
            similar_to: never-written.png

When #

${atago} run missing.atago.yaml

Then #

  • exit code is 1
  • stdout contains could not read baseline image, scenario workdir

atago self-hosting / init #

Source: test/e2e/atago/init.atago.yaml

Scenario: init scaffolds a runnable spec #

When #

${atago} init starter.atago.yaml
${atago} run starter.atago.yaml

Then #

  • after ${atago} init starter.atago.yaml:
    • exit code is 0
    • file starter.atago.yaml exists
  • after ${atago} run starter.atago.yaml:
    • exit code is 0
    • stdout contains PASSED

Generated artifacts #

  • starter.atago.yaml

Scenario: init emits a resolvable schema header for editor completion #

When #

${atago} init headed.atago.yaml
head -1 headed.atago.yaml

Then #

  • after ${atago} init headed.atago.yaml:
    • exit code is 0
  • after head -1 headed.atago.yaml:
    • exit code is 0
    • stdout contains # yaml-language-server: $schema=https://
    • stdout does not contain ./schema/

Scenario: init refuses to overwrite without –force #

Given #

  • Fixture file taken.atago.yaml is created.

Inputs #

Fixture taken.atago.yaml:

version: "1"
suite:
  name: keep
scenarios:
  - name: keep me
    steps:
      - run:
          shell: true
          command: "exit 0"

When #

${atago} init taken.atago.yaml

Then #

  • exit code is 3
  • stderr contains already exists

atago self-hosting / init templates #

Source: test/e2e/atago/init_templates.atago.yaml

Scenario: every template scaffolds a schema-valid spec [template=cli] #

When #

${atago} init --template cli gen.atago.yaml
${atago} explain gen.atago.yaml

Then #

  • after ${atago} init --template cli gen.atago.yaml:
    • exit code is 0
    • file gen.atago.yaml exists
  • after ${atago} explain gen.atago.yaml:
    • exit code is 0

Generated artifacts #

  • gen.atago.yaml

Scenario: every template scaffolds a schema-valid spec [template=http] #

When #

${atago} init --template http gen.atago.yaml
${atago} explain gen.atago.yaml

Then #

  • after ${atago} init --template http gen.atago.yaml:
    • exit code is 0
    • file gen.atago.yaml exists
  • after ${atago} explain gen.atago.yaml:
    • exit code is 0

Generated artifacts #

  • gen.atago.yaml

Scenario: every template scaffolds a schema-valid spec [template=db] #

When #

${atago} init --template db gen.atago.yaml
${atago} explain gen.atago.yaml

Then #

  • after ${atago} init --template db gen.atago.yaml:
    • exit code is 0
    • file gen.atago.yaml exists
  • after ${atago} explain gen.atago.yaml:
    • exit code is 0

Generated artifacts #

  • gen.atago.yaml

Scenario: every template scaffolds a schema-valid spec [template=grpc] #

When #

${atago} init --template grpc gen.atago.yaml
${atago} explain gen.atago.yaml

Then #

  • after ${atago} init --template grpc gen.atago.yaml:
    • exit code is 0
    • file gen.atago.yaml exists
  • after ${atago} explain gen.atago.yaml:
    • exit code is 0

Generated artifacts #

  • gen.atago.yaml

Scenario: every template scaffolds a schema-valid spec [template=ssh] #

When #

${atago} init --template ssh gen.atago.yaml
${atago} explain gen.atago.yaml

Then #

  • after ${atago} init --template ssh gen.atago.yaml:
    • exit code is 0
    • file gen.atago.yaml exists
  • after ${atago} explain gen.atago.yaml:
    • exit code is 0

Generated artifacts #

  • gen.atago.yaml

Scenario: every template scaffolds a schema-valid spec [template=browser] #

When #

${atago} init --template browser gen.atago.yaml
${atago} explain gen.atago.yaml

Then #

  • after ${atago} init --template browser gen.atago.yaml:
    • exit code is 0
    • file gen.atago.yaml exists
  • after ${atago} explain gen.atago.yaml:
    • exit code is 0

Generated artifacts #

  • gen.atago.yaml

Scenario: every template scaffolds a schema-valid spec [template=services] #

When #

${atago} init --template services gen.atago.yaml
${atago} explain gen.atago.yaml

Then #

  • after ${atago} init --template services gen.atago.yaml:
    • exit code is 0
    • file gen.atago.yaml exists
  • after ${atago} explain gen.atago.yaml:
    • exit code is 0

Generated artifacts #

  • gen.atago.yaml

Scenario: list-templates names every runner family with a description #

When #

${atago} init --list-templates

Then #

  • exit code is 0
  • stdout contains cli, http, db, grpc, ssh, browser, services
  • stdout contains runs as-is, edit base_url first

Scenario: unknown template is a configuration error #

When #

${atago} init --template nope gen.atago.yaml

Then #

  • exit code is 3
  • stderr contains unknown template

Scenario: the default cli template runs green #

When #

${atago} init --template cli cli.atago.yaml
${atago} run cli.atago.yaml

Then #

  • after ${atago} init --template cli cli.atago.yaml:
    • exit code is 0
  • after ${atago} run cli.atago.yaml:
    • exit code is 0

Scenario: the db template runs green with the bundled sqlite driver #

When #

${atago} init --template db db.atago.yaml
${atago} run db.atago.yaml

Then #

  • after ${atago} init --template db db.atago.yaml:
    • exit code is 0
  • after ${atago} run db.atago.yaml:
    • exit code is 0

Scenario: the services template runs green and exercises readiness + retry #

skipped on Windows

When #

${atago} init --template services services.atago.yaml
${atago} run services.atago.yaml

Then #

  • after ${atago} init --template services services.atago.yaml:
    • exit code is 0
  • after ${atago} run services.atago.yaml:
    • exit code is 0

atago self-hosting / json numeric comparators #

Source: test/e2e/atago/json_compare.atago.yaml

Scenario: gt and gte pass on a value at or above the bound #

When #

echo '{"count":3,"rate":0.5}'

Then #

  • stdout at $.count is > 2
  • stdout at $.count is >= 3

Scenario: lt and lte pass on a value at or below the bound #

When #

echo '{"count":3,"rate":0.5}'

Then #

  • stdout at $.rate is < 1
  • stdout at $.count is <= 3

Scenario: comparators work on a numeric string #

When #

echo '{"n":"7"}'

Then #

  • stdout at $.n is >= 7

Scenario: comparators apply to rows and file json targets too #

Given #

  • Fixture file metrics.json is created.

Inputs #

Fixture metrics.json:

{"processed": 1200, "errors": 0}

Then #

  • file metrics.json at $.processed is > 1000
  • file metrics.json at $.errors is <= 0

Scenario: a value below the gt bound fails the inner spec #

Given #

  • Fixture file cmp.atago.yaml is created.

Inputs #

Fixture cmp.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: count must exceed 5
    steps:
      - run:
          shell: true
          command: "echo '{\"count\":3}'"
      - assert:
          stdout:
            json:
              path: "$.count"
              gt: 5

When #

${atago} run cmp.atago.yaml

Then #

  • exit code is 1
  • stdout contains is not gt

Scenario: a non-numeric value cannot be compared and fails #

Given #

  • Fixture file cmp.atago.yaml is created.

Inputs #

Fixture cmp.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: name is not a number
    steps:
      - run:
          shell: true
          command: "echo '{\"name\":\"alice\"}'"
      - assert:
          stdout:
            json:
              path: "$.name"
              gt: 0

When #

${atago} run cmp.atago.yaml

Then #

  • exit code is 1
  • stdout contains not numeric

atago self-hosting / json and yaml matcher lists (#156) #

Source: test/e2e/atago/json_list.atago.yaml

Scenario: a file json list asserts several paths at once #

Given #

  • Fixture file starters.json is created.

Inputs #

Fixture starters.json:

[
  {"name": "basei-starter", "default": true},
  {"name": "spec-starter"},
  {"name": "spec87bcd-starter"}
]

Then #

  • file starters.json at $[0].name equals basei-starter; at $[0].default equals true; at $[2].name equals spec87bcd-starter

Scenario: a single mapping still works (backward compatible) #

Given #

  • Fixture file one.json is created.

Inputs #

Fixture one.json:

{"id": 7}

Then #

  • file one.json at $.id equals 7

Scenario: a json list fails the inner spec when one listed path mismatches #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: one listed path is wrong
    steps:
      - fixture:
          file: data.json
          content: '{"a": 1, "b": 2}'
      - assert:
          file:
            path: data.json
            json:
              - { path: "$.a", equals: 1 }
              - { path: "$.b", equals: 999 }

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 1
  • stdout contains did not equal

Scenario: a stdout json list against a JSON-producing command #

skipped on Windows

When #

echo '{"count": 3, "name": "ok"}'

Then #

  • stdout at $.count is >= 2; at $.name equals ok

Scenario: a yaml list asserts several paths on one document #

skipped on Windows

When #

printf 'name: ada\nid: 42\n'

Then #

  • stdout YAML at $.name equals ada; YAML at $.id equals 42

atago self-hosting / json matcher boundary values #

Source: test/e2e/atago/json_matcher_edges.atago.yaml

Scenario: an array element is addressable by index #

When #

printf '{"items":[10,20,30]}'

Then #

  • stdout at $.items[0] equals 10; at $.items[2] equals 30

Scenario: a top-level array reports its length #

When #

printf '[1,2,3,4,5]'

Then #

  • stdout at $ has length 5

Scenario: an empty array has length zero #

When #

printf '{"rows":[]}'

Then #

  • stdout at $.rows has length 0

Scenario: the numeric comparators bound a value #

When #

printf '{"n":50}'

Then #

  • stdout at $.n is > 49; at $.n is >= 50; at $.n is <= 50; at $.n is < 51

Scenario: a boolean value compares equal #

When #

printf '{"ok":true,"off":false}'

Then #

  • stdout at $.ok equals true; at $.off equals false

Scenario: a floating-point value compares equal #

When #

printf '{"pi":3.14}'

Then #

  • stdout at $.pi equals 3.14

Scenario: a string carrying a quote compares equal #

Given #

  • Fixture file quoted.json is created.

Inputs #

Fixture quoted.json:

{"s":"a\"b"}

When #

cat quoted.json

Then #

  • stdout at $.s equals a"b

Scenario: a deeply nested path resolves #

When #

printf '{"x":{"y":{"z":{"w":42}}}}'

Then #

  • stdout at $.x.y.z.w equals 42

Scenario: a path that selects nothing fails with a clear message #

Given #

  • Fixture file nopath.atago.yaml is created.

Inputs #

Fixture nopath.atago.yaml:

version: "1"
suite: {name: nopath}
scenarios:
  - name: an absent key is a clean failure
    steps:
      - run: {shell: true, command: 'printf ''{"a":1}'''}
      - assert: {stdout: {json: [{path: "$.missing", equals: 1}]}}

When #

${atago} run nopath.atago.yaml

Then #

  • exit code is 1
  • stdout contains selected no value

Scenario: a type mismatch failure distinguishes a string from a boolean #

Given #

  • Fixture file typed.atago.yaml is created.

Inputs #

Fixture typed.atago.yaml:

version: "1"
suite: {name: typed}
scenarios:
  - name: string spelling of a boolean
    steps:
      - run: {shell: true, command: 'printf ''{"b":true}'''}
      - assert: {stdout: {json: [{path: "$.b", equals: "true"}]}}

When #

${atago} run typed.atago.yaml

Then #

  • exit code is 1
  • stdout contains $.b == "true", $.b = true

Scenario: a whitespace-only mismatch shows the surrounding spaces #

Given #

  • Fixture file spaced.atago.yaml is created.

Inputs #

Fixture spaced.atago.yaml:

version: "1"
suite: {name: spaced}
scenarios:
  - name: padded value
    steps:
      - run: {shell: true, command: 'printf ''{"v":" x "}'''}
      - assert: {stdout: {json: [{path: "$.v", equals: "x"}]}}

When #

${atago} run spaced.atago.yaml

Then #

  • exit code is 1
  • stdout contains $.v = " x "

Scenario: a numeric mismatch stays unquoted #

Given #

  • Fixture file nums.atago.yaml is created.

Inputs #

Fixture nums.atago.yaml:

version: "1"
suite: {name: nums}
scenarios:
  - name: off by one
    steps:
      - run: {shell: true, command: 'printf ''{"n":2}'''}
      - assert: {stdout: {json: [{path: "$.n", equals: 1}]}}

When #

${atago} run nums.atago.yaml

Then #

  • exit code is 1
  • stdout contains $.n == 1, $.n = 2
  • stdout does not contain "2"

Scenario: a null value is spelled null, not the Go zero value #

When #

printf '{"v": null}'

Then #

  • stdout at $.v matches /^null$/

Scenario: a failure against a null value names it as null #

Given #

  • Fixture file nulled.atago.yaml is created.

Inputs #

Fixture nulled.atago.yaml:

version: "1"
suite: {name: nulled}
scenarios:
  - name: the field is null, the expectation is a string
    steps:
      - run: {shell: true, command: 'printf ''{"v": null}'''}
      - assert: {stdout: {json: [{path: "$.v", equals: "something"}]}}

When #

${atago} run nulled.atago.yaml

Then #

  • exit code is 1
  • stdout contains $.v = null
  • stdout does not contain <nil>

Scenario: equals null asserts the field is null #

When #

printf '{"done": null, "count": 0}'

Then #

  • stdout at $.done equals null; at $.count equals 0

Scenario: equals null rejects every value that is not null #

Given #

  • Fixture file notnull.atago.yaml is created.

Inputs #

Fixture notnull.atago.yaml:

version: "1"
suite: {name: notnull}
scenarios:
  - name: the string spelling of null
    steps:
      - run: {shell: true, command: 'printf ''{"v": "null"}'''}
      - assert: {stdout: {json: {path: "$.v", equals: null}}}
  - name: zero is not null
    steps:
      - run: {shell: true, command: 'printf ''{"v": 0}'''}
      - assert: {stdout: {json: {path: "$.v", equals: null}}}
  - name: false is not null
    steps:
      - run: {shell: true, command: 'printf ''{"v": false}'''}
      - assert: {stdout: {json: {path: "$.v", equals: null}}}
  - name: the empty string is not null
    steps:
      - run: {shell: true, command: 'printf ''{"v": ""}'''}
      - assert: {stdout: {json: {path: "$.v", equals: null}}}

When #

${atago} run notnull.atago.yaml

Then #

  • exit code is 1
  • stdout contains 0 passed, 4 failed, $.v == null, $.v = "null", $.v = 0, $.v = false, $.v = ""

Scenario: an absent path is not null either #

Given #

  • Fixture file absent.atago.yaml is created.

Inputs #

Fixture absent.atago.yaml:

version: "1"
suite: {name: absent}
scenarios:
  - name: the field is not in the document
    steps:
      - run: {shell: true, command: 'printf ''{"other": 1}'''}
      - assert: {stdout: {json: {path: "$.v", equals: null}}}

When #

${atago} run absent.atago.yaml

Then #

  • exit code is 1
  • stdout contains selected no value

Scenario: a json check with no matcher at all is still rejected #

Given #

  • Fixture file bare.atago.yaml is created.

Inputs #

Fixture bare.atago.yaml:

version: "1"
suite: {name: bare}
scenarios:
  - name: a path with nothing to assert
    steps:
      - run: {shell: true, command: 'printf ''{"v": null}'''}
      - assert: {stdout: {json: {path: "$.v"}}}

When #

${atago} run bare.atago.yaml

Then #

  • exit code is 2
  • stderr contains must set one of equals/matches/length/gt/gte/lt/lte

Scenario: explain and doc describe a null assertion as one #

Given #

  • Fixture file nullspec.atago.yaml is created.

Inputs #

Fixture nullspec.atago.yaml:

version: "1"
suite: {name: nullspec}
scenarios:
  - name: a cleared field
    steps:
      - run: {shell: true, command: 'printf ''{"v": null}'''}
      - assert: {stdout: {json: {path: "$.v", equals: null}}}

When #

${atago} explain nullspec.atago.yaml
${atago} doc nullspec.atago.yaml

Then #

  • after ${atago} explain nullspec.atago.yaml:
    • exit code is 0
    • stdout contains stdout JSON $.v == null
    • stdout does not contain <nil>
  • after ${atago} doc nullspec.atago.yaml:
    • exit code is 0
    • stdout contains equals `null`

atago self-hosting / line selector #

Source: test/e2e/atago/line.atago.yaml

Scenario: line selector narrows stdout to a single 1-based line #

When #

printf '[\n  {"id":1}\n]\n'

Then #

  • stdout line 1 equals an exact value
  • stdout line 2 contains "id":1
  • stdout line 3 equals an exact value

Scenario: a trailing newline does not add a phantom final line #

When #

printf 'only-line\n'

Then #

  • stdout line 1 equals an exact value

Scenario: an out-of-range line fails the inner spec #

Given #

  • Fixture file oor.atago.yaml is created.

Inputs #

Fixture oor.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: line 5 of a one-line stream
    steps:
      - run:
          command: echo single
      - assert:
          stdout:
            line: 5
            equals: nope

When #

${atago} run oor.atago.yaml

Then #

  • exit code is 1
  • stdout contains out of range

atago self-hosting / stream text matchers fold CRLF #

Source: test/e2e/atago/line_endings.atago.yaml

Scenario: equals folds a CRLF body to its LF form #

When #

printf 'first\r\nsecond\r\n'

Then #

  • stdout equals an exact value

Expected output #

expected stdout:

first
second

Scenario: equals tolerates the phantom trailing CRLF #

When #

printf 'only\r\n'

Then #

  • stdout equals an exact value

Scenario: contains folds CRLF for a multi-line needle #

When #

printf 'alpha\r\nbeta\r\ngamma\r\n'

Then #

  • stdout contains "alpha\nbeta"

Scenario: contains authored with CRLF matches LF-folded output #

When #

printf 'alpha\r\nbeta\r\n'

Then #

  • stdout contains "alpha\r\nbeta"

Scenario: contains list every multi-line element folds #

When #

printf 'a\r\nb\r\nc\r\nd\r\n'

Then #

  • stdout contains "a\nb", "c\nd"

Scenario: matches anchors a line over CRLF with the multiline flag #

When #

printf 'hello\r\nworld\r\n'

Then #

  • stdout matches /(?m)^world$/

Scenario: matches a literal newline in the pattern over CRLF #

When #

printf 'up\r\ndown\r\n'

Then #

  • stdout matches /up\ndown/

Scenario: not_contains stays clear of an absent multi-line needle #

When #

printf 'red\r\ngreen\r\n'

Then #

  • stdout does not contain "red\nblue"

Scenario: not_matches passes for an anchored line that is absent #

When #

printf 'north\r\nsouth\r\n'

Then #

  • stdout does not match /(?m)^east$/

Scenario: the line selector strips the trailing CR #

When #

printf 'header\r\npayload\r\n'

Then #

  • stdout line 2 equals an exact value

Scenario: json parses a CRLF-formatted document #

When #

printf '{\r\n"count":3\r\n}\r\n'

Then #

  • stdout at $.count equals 3

Scenario: folding does not make an absent multi-line needle match #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: an absent multi-line needle still fails
    steps:
      - run:
          shell: true
          command: printf 'one\r\ntwo\r\n'
      - assert:
          stdout:
            contains: "one\nMISSING"

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains was not present

atago self-hosting / list #

Source: test/e2e/atago/list.atago.yaml

Scenario: list surfaces suites, scenarios, tags, and gates #

Given #

  • Fixture file sample.atago.yaml is created.

Inputs #

Fixture sample.atago.yaml:

version: "1"
suite:
  name: demo
scenarios:
  - name: tagged scenario
    tags: [smoke, fast]
    steps:
      - run: {command: "true"}
      - assert: {exit_code: 0}
  - name: gated scenario
    skip: {env: ATAGO_SKIP_DEMO}
    steps:
      - run: {shell: true, command: "echo hi > out.txt"}
      - assert: {file: {path: out.txt, exists: true}}

When #

${atago} list sample.atago.yaml

Then #

  • exit code is 0
  • stdout contains tagged scenario, smoke, skip:env=ATAGO_SKIP_DEMO

Scenario: list –json is a stable machine contract #

Given #

  • Fixture file sample.atago.yaml is created.

Inputs #

Fixture sample.atago.yaml:

version: "1"
suite:
  name: demo
scenarios:
  - name: only scenario
    steps:
      - run: {command: "true"}
      - assert: {exit_code: 0}

When #

${atago} list --json sample.atago.yaml

Then #

  • exit code is 0
  • stdout at $.schema_version equals 1
  • stdout at $.scenarios[0].scenario equals only scenario

Scenario: list marks an expect_fail scenario #

Given #

  • Fixture file xfail.atago.yaml is created.

Inputs #

Fixture xfail.atago.yaml:

version: "1"
suite:
  name: xfail
scenarios:
  - name: known bug
    expect_fail:
      reason: "upstream renders the wrong width"
      issue: "https://example.com/issues/42"
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}
  - name: healthy
    steps:
      - run: {shell: true, command: "true"}
      - assert: {exit_code: 0}

When #

${atago} list xfail.atago.yaml
${atago} list --json xfail.atago.yaml

Then #

  • after ${atago} list xfail.atago.yaml:
    • exit code is 0
    • stdout contains XFAIL, known bug
  • after ${atago} list --json xfail.atago.yaml:
    • exit code is 0
    • stdout at $.scenarios[0].expect_fail.reason equals upstream renders the wrong width; at $.scenarios[0].expect_fail.issue equals https://example.com/issues/42

atago self-hosting / loader rejects malformed specs #

Source: test/e2e/atago/loader_errors.atago.yaml

Scenario: an empty scenario list is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: []

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains must contain at least one scenario

Scenario: a wrong version string is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "9"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains version must be "1"

Scenario: an unknown top-level field is rejected with its position #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenariosss: []

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains unknown field "scenariosss"

Scenario: a step that sets two actions is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
        fixture: {file: f, content: c}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains must set exactly one action

Scenario: a stream assertion with no matcher is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains must set at least one matcher

Scenario: combining equals with another matcher is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {equals: hi, contains: h}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains cannot be combined with another matcher

Scenario: a line index below one is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {line: 0, equals: x}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains line must be >= 1

Scenario: combining a line selector with json is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {line: 1, json: [{path: "$.k", equals: 1}]}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains cannot be combined with json/yaml/snapshot

Scenario: a duplicate scenario name is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - {name: dup, steps: [{run: {command: echo}}]}
  - {name: dup, steps: [{run: {command: echo}}]}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains duplicate scenario name "dup"

Scenario: an empty run command is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: ""}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains run.command is required

Scenario: an unparseable timeout is rejected with an example #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: echo, timeout: "soon"}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains is not a valid duration

Scenario: a fixture with two content sources is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{fixture: {file: f, content: c, base64: "QQ=="}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains set only one of content, base64, from, or symlink

Scenario: an absolute changes glob is rejected as not workdir-relative #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {changes: {created: ["/etc/passwd"]}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains must be workdir-relative, not absolute

Scenario: the inline stdin form is a scalar, not a mapping key #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios: [{name: a, steps: [{run: {command: cat, stdin: {inline: hi}}}]}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains unknown key "inline"

Scenario: a wrong-typed exit_code is rejected with its position and excerpt #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert:
          exit_code: zero

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains [8:22], exit_code must be an integer, exit_code: zero

Scenario: a missing target names the reason without syscall noise #

When #

${atago} run no-such-spec.atago.yaml

Then #

  • exit code is 3
  • stderr contains cannot access "no-such-spec.atago.yaml":
  • stderr does not contain stat no-such-spec

Scenario: an empty directory says how to create a first spec #

When #

mkdir specs
${atago} run specs

Then #

  • after ${atago} run specs:
    • exit code is 3
    • stderr contains no *.atago.yaml, specs, atago init

Scenario: a cwd that traverses out of the workdir is refused before running #

Given #

  • Fixture file escape.atago.yaml is created.
  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture escape.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: runs somewhere else
    steps:
      - run:
          shell: true
          command: pwd
          cwd: "../../../.."

Fixture ok.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: runs in a sub-directory
    steps:
      - run: {shell: true, command: "mkdir -p sub"}
      - run: {shell: true, command: "pwd", cwd: sub}
      - assert:
          exit_code: 0
          stdout:
            contains: sub

When #

${atago} run escape.atago.yaml
${atago} run ok.atago.yaml

Then #

  • after ${atago} run escape.atago.yaml:
    • exit code is 2
    • stderr contains run.cwd, escapes the scenario workdir
  • after ${atago} run ok.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: a gate with no condition is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    only: {}
    steps: [{run: {command: echo}}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains only must name a condition (os, env, or command)

Scenario: skip and only naming the same condition are rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    skip: {os: linux}
    only: {os: linux}
    steps: [{run: {command: echo}}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains skip.os and only.os both name, can never run anywhere

Scenario: gates naming different fields still load #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    skip: {os: windows}
    only: {command: "no-such-tool-zzz"}
    steps: [{run: {command: echo}}]

When #

${atago} run ok.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 skipped

Scenario: an empty deterministic compare list is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps: [{run: {command: echo, deterministic: {compare: []}}}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains deterministic.compare must not be empty

Scenario: an empty-matching pattern under a count bound is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {matches: "q*", max_count: 0}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains matches the empty string

Scenario: an empty-matching store capture is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - store: {name: v, from: {stdout: {matches: "[0-9]*"}}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains captures "" from any output

Scenario: an empty-matching scrub rule is rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scrub:
  - {pattern: "[0-9]*", placeholder: "<ID>"}
scenarios:
  - name: a
    steps: [{run: {command: echo}}]

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains matches between every byte

Scenario: matchers of one assert that contradict each other are rejected #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {stdout: {contains: [abc], not_contains: [abc]}}
  - name: b
    steps:
      - assert: {dir: {path: d, contains: [x], count: 0}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains contains and not_contains both list "abc", count: 0 cannot hold together with contains

Scenario: a file assertion with no matcher names the size bounds too #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {file: {path: out.txt}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains must set one of exists/contains/not_contains/executable/equals/equals_file/json/snapshot/size/min_size/max_size

Scenario: a file assertion with two content matchers still names only those #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert: {file: {path: out.txt, exists: true, snapshot: s.txt}}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains must set exactly one of exists/contains/not_contains/executable/equals/equals_file/json/snapshot

Scenario: a list written where one value belongs names the key and the shape #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run: {command: echo}
      - assert:
          stdout:
            matches:
              - "^h"
              - "i$"

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains "matches" takes a single value, not a list, one assert per pattern

Scenario: a mapping written where text belongs names the key too #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: x}
scenarios:
  - name: a
    steps:
      - run:
          command:
            bin: echo

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains "command" takes a single value, not a mapping

atago self-hosting / manifest #

Source: test/e2e/atago/manifest.atago.yaml

Scenario: manifest emits a stable JSON summary without running the spec #

Given #

  • Fixture file sample.atago.yaml is created.

Inputs #

Fixture sample.atago.yaml:

version: "1"
suite:
  name: demo
secrets:
  - TOKEN
scenarios:
  - name: "greets ${who}"
    tags: [smoke]
    matrix:
      - { who: Alice }
      - { who: Bob }
    steps:
      - run:
          command: echo ${who}
      - assert:
          stdout:
            contains: ${who}
          file:
            path: out.txt
            exists: true

When #

${atago} manifest sample.atago.yaml

Then #

  • exit code is 0
  • stdout at $.schema_version equals 1
  • stdout at $.specs[0].suite equals demo
  • stdout at $.specs[0].secrets[0] equals TOKEN
  • stdout at $.specs[0].scenarios has length 2
  • stdout at $.specs[0].scenarios[0].vars.who equals Alice
  • stdout at $.specs[0].scenarios[0].generates[0] equals out.txt
  • stdout at $.specs[0].source.line equals 3
  • stdout at $.specs[0].scenarios[0].source.line equals 7
  • stdout at $.specs[0].scenarios[1].source.line equals 7
  • stdout at $.specs[0].scenarios[0].steps[0].source.line equals 13

Scenario: manifest does not execute the spec’s commands #

Given #

  • Fixture file side_effect.atago.yaml is created.

Inputs #

Fixture side_effect.atago.yaml:

version: "1"
suite:
  name: side
scenarios:
  - name: would write a file
    steps:
      - run:
          shell: true
          command: touch executed.marker
      - assert:
          exit_code: 0

When #

${atago} manifest side_effect.atago.yaml

Then #

  • exit code is 0
  • file executed.marker does not exist

Scenario: manifest carries the declarative fields of steps and runners #

Given #

  • Fixture file fields.atago.yaml is created.

Inputs #

Fixture fields.atago.yaml:

version: "1"
suite:
  name: fields
runners:
  slowbox: {type: cmd, cwd: ./sub, timeout: 45s}
  jump: {type: ssh, host: "shell.example:2222", user: deploy, password: hunter2, insecure_host_key: true}
scenarios:
  - name: declarative knobs
    steps:
      - run:
          command: build-tool compile
          cwd: sub/dir
          timeout: 90s
          stdout_to: logs/build.log
          stderr_to: logs/build.err

When #

${atago} manifest fields.atago.yaml

Then #

  • exit code is 0
  • stdout at $.specs[0].scenarios[0].steps[0].cwd equals sub/dir; at $.specs[0].scenarios[0].steps[0].timeout equals 90s; at $.specs[0].scenarios[0].steps[0].stdout_to equals logs/build.log; at $.specs[0].scenarios[0].steps[0].stderr_to equals logs/build.err; at $.specs[0].runners[0].insecure_host_key equals true; at $.specs[0].runners[0].user equals deploy; at $.specs[0].runners[1].cwd equals ./sub; at $.specs[0].runners[1].timeout equals 45s
  • stdout does not contain hunter2

Scenario: manifest describes the suite lifecycle outputs and the subject build #

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file suitegen.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

subject:
  name: mytool
  artifact: bin/mytool
  build:
    command: "curl -s https://build.example/prebuilt > $${artifact}"
    shell: true

Fixture suitegen.atago.yaml:

version: "1"
suite:
  name: suitegen
  setup:
    - run: {command: seed, stdout_to: build/seed.txt}
  teardown:
    - run: {command: audit, stdout_to: logs/suite-audit.log}
scenarios:
  - name: quiet
    steps:
      - run: {command: echo}

When #

${atago} manifest suitegen.atago.yaml
${atago} explain suitegen.atago.yaml

Then #

  • after ${atago} manifest suitegen.atago.yaml:
    • exit code is 0
    • stdout at $.specs[0].suite_generates[0] equals build/seed.txt; at $.specs[0].suite_generates[1] equals logs/suite-audit.log; at $.specs[0].subject.name equals mytool; at $.specs[0].subject.shell equals true; at $.specs[0].suite_security[0] equals shell execution enabled (subject build mytool): curl -s https://build.example/prebuilt > $${artifact}
    • stdout contains "project_path"
  • after ${atago} explain suitegen.atago.yaml:
    • exit code is 0
    • stdout contains Subject under test: mytool (built by: curl -s https://build.example/prebuilt > $${artifact}, shell), Suite generates:, build/seed.txt, network access (subject build mytool)

Scenario: an assert step carries the assertion, not only its target #

Given #

  • Fixture file strong.atago.yaml is created.
  • Fixture file weak.atago.yaml is created.

Inputs #

Fixture strong.atago.yaml:

version: "1"
suite: {name: same}
scenarios:
  - name: checks
    steps:
      - run: {command: echo hi}
      - assert:
          exit_code: 0
          stdout: {equals: "hi\n"}

Fixture weak.atago.yaml:

version: "1"
suite: {name: same}
scenarios:
  - name: checks
    steps:
      - run: {command: echo hi}
      - assert:
          exit_code: {in: [0, 1, 2]}
          stdout: {not_contains: "zzz"}

When #

${atago} manifest strong.atago.yaml
${atago} manifest weak.atago.yaml

Then #

  • after ${atago} manifest strong.atago.yaml:
    • exit code is 0
    • file strong.json contains exit code is 0, stdout equals exact text
  • after ${atago} manifest weak.atago.yaml:
    • exit code is 0
    • file weak.json contains exit code in [0, 1, 2], stdout does not contain
    • file weak.json does not contain exit code is 0, stdout equals exact text

Generated artifacts #

  • strong.json
  • weak.json

Scenario: explain and manifest substitute a matrix row into the step text #

Given #

  • Fixture file rows.atago.yaml is created.

Inputs #

Fixture rows.atago.yaml:

version: "1"
suite: {name: rows}
scenarios:
  - name: greets $${who}
    matrix:
      - {who: alice}
      - {who: bob}
    steps:
      - run: {command: "echo $${who}"}
      - assert: {exit_code: 0}

When #

${atago} explain rows.atago.yaml
${atago} manifest rows.atago.yaml

Then #

  • after ${atago} explain rows.atago.yaml:
    • exit code is 0
    • stdout contains echo alice, echo bob, does not contain echo ${who}
  • after ${atago} manifest rows.atago.yaml:
    • exit code is 0
    • stdout contains "command": "echo alice", "command": "echo bob", does not contain "command": "echo ${who}"
    • stdout at $.specs[0].scenarios[0].vars.who equals alice

atago self-hosting / matrix scenarios #

Source: test/e2e/atago/matrix.atago.yaml

Scenario: matrix expands into one scenario per row #

Given #

  • Fixture file matrix.atago.yaml is created.

Inputs #

Fixture matrix.atago.yaml:

version: "1"
suite:
  name: greetings
scenarios:
  - name: "greets ${who}"
    matrix:
      - { who: Alice }
      - { who: Bob }
    steps:
      - run:
          shell: true
          command: echo ${who}
      - assert:
          stdout:
            contains: ${who}

When #

${atago} run --report junit matrix.atago.yaml

Then #

  • exit code is 0
  • stdout contains name="greets Alice", name="greets Bob"

Scenario: matrix without a templated name gets a deterministic suffix #

Given #

  • Fixture file suffix.atago.yaml is created.

Inputs #

Fixture suffix.atago.yaml:

version: "1"
suite:
  name: suffixed
scenarios:
  - name: row
    matrix:
      - { n: "1" }
      - { n: "2" }
    steps:
      - run:
          shell: true
          command: echo ${n}
      - assert:
          stdout:
            contains: ${n}

When #

${atago} run --report junit suffix.atago.yaml

Then #

  • exit code is 0
  • stdout contains name="row [n=1]", name="row [n=2]"

Scenario: stdout_to expands a matrix variable into the redirect target [who=alice] #

When #

printf hello-alice

Then #

  • exit code is 0
  • file out-alice.txt contains hello-alice

Generated artifacts #

  • out-alice.txt

Scenario: stdout_to expands a matrix variable into the redirect target [who=bob] #

When #

printf hello-bob

Then #

  • exit code is 0
  • file out-bob.txt contains hello-bob

Generated artifacts #

  • out-bob.txt

atago self-hosting / matrix expansion boundary values #

Source: test/e2e/atago/matrix_edges.atago.yaml

Scenario: each row substitutes into the scenario name #

Given #

  • Fixture file names.atago.yaml is created.

Inputs #

Fixture names.atago.yaml:

version: "1"
suite: {name: names}
scenarios:
  - name: "case ${n}"
    matrix:
      - {n: "1"}
      - {n: "2"}
      - {n: "3"}
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json names.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 3; at $.suites[0].scenarios[0].name equals case 1; at $.suites[0].scenarios[2].name equals case 3

Scenario: a row with several variables substitutes all of them #

Given #

  • Fixture file multi.atago.yaml is created.

Inputs #

Fixture multi.atago.yaml:

version: "1"
suite: {name: multi}
scenarios:
  - name: "${who} speaks ${lang}"
    matrix:
      - {who: Alice, lang: en}
      - {who: Bob, lang: fr}
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json multi.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios[0].name equals Alice speaks en; at $.suites[0].scenarios[1].name equals Bob speaks fr

Scenario: a single-row matrix expands to exactly one scenario #

Given #

  • Fixture file single.atago.yaml is created.

Inputs #

Fixture single.atago.yaml:

version: "1"
suite: {name: single}
scenarios:
  - name: "only ${k}"
    matrix:
      - {k: solo}
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run --ci --report json single.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].scenarios has length 1; at $.suites[0].scenarios[0].name equals only solo

Scenario: an empty matrix row list is a load-time error #

Given #

  • Fixture file emptymx.atago.yaml is created.

Inputs #

Fixture emptymx.atago.yaml:

version: "1"
suite: {name: emptymx}
scenarios:
  - name: "e ${n}"
    matrix: []
    steps: [{run: {shell: true, command: "exit 0"}}]

When #

${atago} run emptymx.atago.yaml

Then #

  • exit code is 2
  • stderr contains ATG2202: , matrix must contain at least one row

Scenario: rows that expand to the same name are rejected as duplicates #

Given #

  • Fixture file dupmx.atago.yaml is created.

Inputs #

Fixture dupmx.atago.yaml:

version: "1"
suite: {name: dupmx}
scenarios:
  - name: "dup ${n}"
    matrix:
      - {n: same}
      - {n: same}
    steps: [{run: {shell: true, command: "exit 0"}}]

When #

${atago} run dupmx.atago.yaml

Then #

  • exit code is 2
  • stderr contains duplicate scenario name "dup same"

Scenario: a row that leaves a referenced name unbound is a load-time error #

Given #

  • Fixture file unbound.atago.yaml is created.
  • Fixture file escaped.atago.yaml is created.

Inputs #

Fixture unbound.atago.yaml:

version: "1"
suite: {name: unbound}
scenarios:
  - name: "greets ${who}"
    matrix:
      - {who: Alice}
      - {name: Bob}
    steps: [{run: {shell: true, command: "exit 0"}}]

Fixture escaped.atago.yaml:

version: "1"
suite: {name: escaped}
scenarios:
  - name: "writes $$${who} verbatim ${n}"
    matrix:
      - {n: "1"}
    steps: [{run: {shell: true, command: "exit 0"}}, {assert: {exit_code: 0}}]

When #

${atago} run unbound.atago.yaml
${atago} run --ci --report json escaped.atago.yaml

Then #

  • after ${atago} run unbound.atago.yaml:
    • exit code is 2
    • stderr contains ATG2403: , does not bind ${who}
  • after ${atago} run --ci --report json escaped.atago.yaml:
    • exit code is 0
    • stdout at $.suites[0].scenarios[0].name equals writes $$${who} verbatim 1

atago self-hosting / mock http server (offline API-client testing) #

Source: test/e2e/atago/mock_server.atago.yaml

Scenario: count, header, and body-json asserts pass against a real client #

Given #

  • Stub HTTP server api serves 1 canned route(s) at ${api.url} and records every request (#24).
  • Fixture file client.atago.yaml is created.

Inputs #

Fixture client.atago.yaml:

version: "1"
suite:
  name: client
runners:
  api:
    type: http
    base_url: ${api.url}
scenarios:
  - name: post a report
    steps:
      - http:
          runner: api
          method: POST
          path: /v1/reports
          header: { Authorization: "Bearer tok-123" }
          json: { title: "report" }
      - assert:
          status: 201

When #

${atago} run client.atago.yaml

Then #

  • exit code is 0
  • mock api received POST /v1/reports exactly 1 time(s)

Scenario: a failing count summarizes the recorded requests #

Given #

  • Stub HTTP server stub serves 1 canned route(s) at ${stub.url} and records every request (#24).
  • Fixture file outer.atago.yaml is created.

Inputs #

Fixture outer.atago.yaml:

version: "1"
suite:
  name: outer
runners:
  api:
    type: http
    base_url: ${stub.url}
scenarios:
  - name: wrong path then failing count
    mock_servers:
      - name: inner
        routes:
          - method: GET
            path: /right
    steps:
      - http:
          runner: api
          method: GET
          path: /wrong
      - assert:
… (truncated, 6 more lines)

When #

${atago} run outer.atago.yaml

Then #

  • exit code is 1
  • stdout contains 1 request for /right, 0 matching of 0 recorded

Scenario: an unknown mock name in an assert is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: wrong name
    mock_servers:
      - name: api
        routes: [{method: GET, path: /}]
    steps:
      - run: {command: echo hi}
      - assert:
          mock: {name: apo, count: 1}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains not a declared mock server (declared: api)

Scenario: a route that can never answer is a load-time error #

Given #

  • Fixture file dead.atago.yaml is created.
  • Fixture file query.atago.yaml is created.

Inputs #

Fixture dead.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: two routes for one pair
    mock_servers:
      - name: api
        routes:
          - {method: GET, path: /ping, status: 200, body: first}
          - {method: get, path: /ping, status: 500, body: second}
    steps:
      - run: {command: echo hi}

Fixture query.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: a route declaring a query
    mock_servers:
      - name: api
        routes:
          - {method: GET, path: "/search?q=x", status: 200, body: hit}
    steps:
      - run: {command: echo hi}

When #

${atago} run dead.atago.yaml
${atago} run query.atago.yaml

Then #

  • after ${atago} run dead.atago.yaml:
    • exit code is 2
    • stderr contains duplicate route GET /ping
  • after ${atago} run query.atago.yaml:
    • exit code is 2
    • stderr contains must not contain a query string

atago self-hosting / combined stream matchers #

Source: test/e2e/atago/multi_matcher.atago.yaml

Scenario: contains and not_contains hold together #

When #

echo "hello world"

Then #

  • stdout contains hello, does not contain goodbye

Scenario: matches and not_matches hold together #

When #

echo "release 1.2.3"

Then #

  • stdout matches /[0-9]+\.[0-9]+\.[0-9]+/, does not match /(?i)error/

Scenario: all four text matchers compose #

When #

echo "Alice and Bob"

Then #

  • stdout contains Alice, Bob, does not contain Carol, matches /A.+e/, does not match /Dave/

Scenario: a combined matcher composes with a line selector #

When #

printf 'first line\nsecond line\n'

Then #

  • stdout line 2 contains second, does not contain first

Scenario: a failing member fails the inner spec and names the offender #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: contains holds but not_contains does not
    steps:
      - run:
          command: echo "hello goodbye"
      - assert:
          stdout:
            contains: hello
            not_contains: goodbye

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains goodbye, does not contain internal error

Scenario: mixing a whole-stream matcher with a text matcher is a load error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: equals cannot combine
    steps:
      - run:
          command: echo hi
      - assert:
          stdout:
            equals: hi
            contains: h

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains cannot be combined with another matcher

atago self-hosting / not_equals matcher #

Source: test/e2e/atago/not_equals.atago.yaml

Scenario: not_equals passes when stdout differs from the given text #

When #

echo Bob

Then #

  • stdout does not equal an exact value

Scenario: not_equals is trailing-newline tolerant like equals #

When #

echo hello

Then #

  • stdout does not equal an exact value

Scenario: not_equals composes with a line selector #

When #

printf 'first\nsecond\n'

Then #

  • stdout line 2 does not equal an exact value

Scenario: not_equals fails the inner spec when the text matches exactly #

Given #

  • Fixture file ne.atago.yaml is created.

Inputs #

Fixture ne.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: stdout equals the forbidden text
    steps:
      - run:
          command: echo same
      - assert:
          stdout:
            not_equals: same

When #

${atago} run ne.atago.yaml

Then #

  • exit code is 1
  • stdout contains unexpectedly equaled

atago self-hosting / parallel #

Source: test/e2e/atago/parallel.atago.yaml

Scenario: parallel run passes and stays deterministic #

Given #

  • Fixture file many.atago.yaml is created.

Inputs #

Fixture many.atago.yaml:

version: "1"
suite:
  name: many
scenarios:
  - name: a
    steps: [{run: {shell: true, command: echo a}}, {assert: {stdout: {contains: a}}}]
  - name: b
    steps: [{run: {shell: true, command: echo b}}, {assert: {stdout: {contains: b}}}]
  - name: c
    steps: [{run: {shell: true, command: echo c}}, {assert: {stdout: {contains: c}}}]

When #

${atago} run --parallel 3 many.atago.yaml

Then #

  • exit code is 0
  • stdout contains 3 passed

Scenario: fail-fast stops after the first failure #

Given #

  • Fixture file ff.atago.yaml is created.

Inputs #

Fixture ff.atago.yaml:

version: "1"
suite:
  name: ff
scenarios:
  - name: boom
    steps: [{run: {shell: true, command: "exit 1"}}, {assert: {exit_code: 0}}]
  - name: never
    steps: [{run: {shell: true, command: echo hi}}, {assert: {exit_code: 0}}]

When #

${atago} run --parallel 1 --fail-fast ff.atago.yaml

Then #

  • exit code is 1
  • stdout contains 1 skipped

Scenario: scenario-scoped services stay isolated under parallel workers #

skipped on Windows

Given #

  • Fixture file par-svc.atago.yaml is created.

Inputs #

Fixture par-svc.atago.yaml:

version: "1"
suite:
  name: par-svc
scenarios:
  - name: one
    services:
      - name: peer1
        shell: true
        command: 'echo token-1 > addr.txt; sleep 30'
        ready: {file: addr.txt, store: tok, timeout: 5s}
    steps:
      - run: {shell: true, command: "echo got ${tok}"}
      - assert: {exit_code: 0, stdout: {contains: got token-1}}
  - name: two
    services:
      - name: peer2
        shell: true
        command: 'echo token-2 > addr.txt; sleep 30'
        ready: {file: addr.txt, store: tok, timeout: 5s}
    steps:
… (truncated, 11 more lines)

When #

${atago} run --parallel 3 par-svc.atago.yaml

Then #

  • exit code is 0
  • stdout contains 3 passed

atago self-hosting / forward-slash spec paths resolve on every OS #

Source: test/e2e/atago/paths_portable.atago.yaml

Scenario: stdout_to creates a nested parent directory #

When #

echo produced

Then #

  • exit code is 0
  • file out/logs/result.txt contains produced

Generated artifacts #

  • out/logs/result.txt

Scenario: stderr_to creates its own nested parent directory #

When #

echo oops 1>&2

Then #

  • exit code is 0
  • file errs/deep/err.txt contains oops

Generated artifacts #

  • errs/deep/err.txt

Scenario: a fixture at a nested forward-slash path is created and addressable #

Given #

  • Fixture file data/config/app.json is created.

Inputs #

Fixture data/config/app.json:

{"k":1}

Then #

  • file data/config/app.json at $.k equals 1

Scenario: a file assert reaches a deeply nested fixture by forward-slash path #

Given #

  • Fixture file a/b/c/leaf.txt is created.

Inputs #

Fixture a/b/c/leaf.txt:

at the bottom

Then #

  • file a/b/c/leaf.txt contains at the bottom

Scenario: a dir assert addresses a nested tree and child by forward-slash path #

Given #

  • Fixture file pkg/mod/one.go is created.
  • Fixture file pkg/mod/two.go is created.
  • Fixture file pkg/mod/sub/three.go is created.

Inputs #

Fixture pkg/mod/one.go:

package mod

Fixture pkg/mod/two.go:

package mod

Fixture pkg/mod/sub/three.go:

package sub

Then #

  • dir pkg/mod exists, contains one.go, contains two.go, contains sub/three.go, does not contain missing.go

Scenario: equals_file compares two files addressed by forward-slash paths #

Given #

  • Fixture file golden/expected.bin is created.
  • Fixture file build/actual.bin is created.

Then #

  • file build/actual.bin is byte-identical to golden/expected.bin

Scenario: a redirect path may not escape the workdir via a nested traversal #

Given #

  • Fixture file probe.atago.yaml is created.

Inputs #

Fixture probe.atago.yaml:

version: "1"
suite:
  name: escape
scenarios:
  - name: nested traversal is rejected
    steps:
      - run:
          shell: true
          command: echo x
          stdout_to: sub/../../escape.txt

When #

${atago} run probe.atago.yaml

Then #

  • exit code is 4
  • stdout contains escapes the scenario workdir

atago self-hosting / pdf assertion #

Source: test/e2e/atago/pdf.atago.yaml

Scenario: pdf assertions cover page count, metadata, and text #

Given #

  • Fixture file report.pdf is created.

Inputs #

Fixture report.pdf:

%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>
endobj
4 0 obj
<< /Length 46 >>
stream
BT /F1 24 Tf 72 700 Td (Hello atago report) Tj ET
endstream
endobj
6 0 obj
<< /Title (Quarterly Report) /Author (atago) >>
endobj
trailer
… (truncated, 2 more lines)

Then #

  • pdf report.pdf 1 page, >= 1 page, <= 3 pages, author contains atago, title contains Quarterly, text contains Hello atago report

Generated artifacts #

  • report.pdf

Scenario: a non-pdf file fails the pdf target #

Given #

  • Fixture file notpdf.txt is created.

Inputs #

Fixture notpdf.txt:

just text

When #

${atago} version

Then #

  • exit code is 0

Scenario: metadata is found inside a compressed object stream #

Given #

  • Fixture file objstm.atago.yaml is created.

Inputs #

Fixture objstm.atago.yaml:

version: "1"
suite: {name: objstm}
scenarios:
  - name: the title lives in a Flate-compressed object stream
    steps:
      - fixture:
          file: compressed.pdf
          base64: JVBERi0xLjUKMSAwIG9iago8PCAvVHlwZSAvUGFnZSA+PgplbmRvYmoKMiAwIG9iago8PCAvVHlwZSAvT2JqU3RtIC9OIDEgL0ZpcnN0IDAgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCA1MCA+PgpzdHJlYW0KeJyzsdEPySzJSdVwzs8tKEotLk5NUQALaOo7lpZk5BdpBJcUpSbmKkB4mnZ2AMDhEaAKZW5kc3RyZWFtCmVuZG9iagp0cmFpbGVyCjw8IC9Sb290IDEgMCBSID4+CiUlRU9GCg==
      - run: {command: echo produced}
      - assert:
          pdf:
            path: compressed.pdf
            metadata:
              title: Compressed Title
              author: Stream Author

When #

${atago} run objstm.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: a wrong expectation still fails against compressed metadata #

Given #

  • Fixture file wrongmeta.atago.yaml is created.

Inputs #

Fixture wrongmeta.atago.yaml:

version: "1"
suite: {name: wrongmeta}
scenarios:
  - name: the title is not what the spec claims
    steps:
      - fixture:
          file: compressed.pdf
          base64: JVBERi0xLjUKMSAwIG9iago8PCAvVHlwZSAvUGFnZSA+PgplbmRvYmoKMiAwIG9iago8PCAvVHlwZSAvT2JqU3RtIC9OIDEgL0ZpcnN0IDAgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCA1MCA+PgpzdHJlYW0KeJyzsdEPySzJSdVwzs8tKEotLk5NUQALaOo7lpZk5BdpBJcUpSbmKkB4mnZ2AMDhEaAKZW5kc3RyZWFtCmVuZG9iagp0cmFpbGVyCjw8IC9Sb290IDEgMCBSID4+CiUlRU9GCg==
      - run: {command: echo produced}
      - assert:
          pdf:
            path: compressed.pdf
            metadata:
              title: Not This Title

When #

${atago} run wrongmeta.atago.yaml

Then #

  • exit code is 1
  • stdout contains Compressed Title

Scenario: a stream that ends without a newline does not swallow the next object #

Given #

  • Fixture file nonewline.atago.yaml is created.

Inputs #

Fixture nonewline.atago.yaml:

version: "1"
suite: {name: nonewline}
scenarios:
  - name: both pages contribute their text
    steps:
      - fixture:
          file: two-pages.pdf
          base64: JVBERi0xLjUKMSAwIG9iago8PCAvVHlwZSAvUGFnZSA+PgplbmRvYmoKMiAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJxzClHQcMssKi5RKEhMT1Uozy9KKdZUCMlScA0BAHn8CLNlbmRzdHJlYW0KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCniccwpR0AhOTc7PS1EoSExPVSjPL0op1lQIyVJwDQEAgxUJBwplbmRzdHJlYW0KZW5kb2JqCnRyYWlsZXIKPDwgL1Jvb3QgMSAwIFIgPj4KJSVFT0YK
      - run: {command: echo produced}
      - assert:
          pdf:
            path: two-pages.pdf
            pages: 2
      - assert:
          pdf:
            path: two-pages.pdf
            text:
              contains: ["First 
… (truncated)

When #

${atago} run nonewline.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

atago self-hosting / directory manifest #

Source: test/e2e/atago/project_manifest.atago.yaml

Scenario: a manifest applies to a spec nested below it #

skipped on Windows

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file corpus/golden.txt is created.
  • Fixture file deep/nested/inner.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

env:
  PROJECT_WIDE: from-manifest
fixtures_dir: corpus

Fixture corpus/golden.txt:

committed corpus

Fixture deep/nested/inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: reads the manifest env and the fixture corpus
    steps:
      - run:
          shell: true
          command: "echo v=$PROJECT_WIDE && cat $${fixtures}/golden.txt"
      - assert:
          stdout:
            contains:
              - "v=from-manifest"
              - "committed corpus"

When #

${atago} run deep/nested/inner.atago.yaml

Then #

  • exit code is 0

Scenario: the nearest manifest wins #

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file deep/atago.project.yaml is created.
  • Fixture file deep/spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

env:
  WHICH: outer

Fixture deep/atago.project.yaml:

env:
  WHICH: inner

Fixture deep/spec.atago.yaml:

version: "1"
suite:
  name: nearest
scenarios:
  - name: reads the nearer manifest
    steps:
      - run:
          shell: true
          command: "echo which=$WHICH"
      - assert:
          stdout:
            contains: "which=inner"

When #

${atago} run deep/spec.atago.yaml

Then #

  • exit code is 0

Scenario: a spec’s own values beat the manifest #

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

env:
  SHARED: manifest
  ONLY_HERE: yes
defaults:
  run:
    shell: true

Fixture spec.atago.yaml:

version: "1"
suite:
  name: precedence
  env:
    SHARED: spec
scenarios:
  - name: the spec wins per key, and inherits the rest
    steps:
      - run:
          command: "echo shared=$SHARED only=$ONLY_HERE"
      - assert:
          stdout:
            contains: "shared=spec only=yes"

When #

${atago} run spec.atago.yaml

Then #

  • exit code is 0

Scenario: specdir points at the spec’s own directory #

skipped on Windows

Given #

  • Fixture file sub/marker.txt is created.
  • Fixture file sub/spec.atago.yaml is created.

Inputs #

Fixture sub/marker.txt:

beside the spec

Fixture sub/spec.atago.yaml:

version: "1"
suite:
  name: specdir
scenarios:
  - name: reads a file beside itself
    steps:
      - run:
          shell: true
          command: "cat $${specdir}/marker.txt"
      - assert:
          stdout:
            contains: "beside the spec"

When #

${atago} run sub/spec.atago.yaml

Then #

  • exit code is 0

Scenario: explain names the manifest that applied #

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file corpus/keep.txt is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

env:
  SHOWN: yes
fixtures_dir: corpus

Fixture corpus/keep.txt:

x

Fixture spec.atago.yaml:

version: "1"
suite:
  name: shown
scenarios:
  - name: one
    steps:
      - run:
          command: echo hi

When #

${atago} explain spec.atago.yaml

Then #

  • exit code is 0
  • stdout contains Project manifest:, atago.project.yaml, Fixtures ($${fixtures}):

Scenario: a manifest pointing at a missing fixtures dir fails to load #

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

fixtures_dir: not-there

Fixture spec.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: one
    steps:
      - run:
          command: echo hi

When #

${atago} run spec.atago.yaml

Then #

  • exit code is 2
  • stderr contains fixtures_dir "not-there" does not exist

Scenario: an unknown manifest key is rejected #

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

scenarios: []

Fixture spec.atago.yaml:

version: "1"
suite:
  name: bad key
scenarios:
  - name: one
    steps:
      - run:
          command: echo hi

When #

${atago} run spec.atago.yaml

Then #

  • exit code is 2
  • stderr contains scenarios

atago self-hosting / pty #

Source: test/e2e/atago/pty.atago.yaml

Scenario: a pty step sees a terminal where a run step sees a pipe #

skipped on Windows

Given #

  • Fixture file tty.atago.yaml is created.

Inputs #

Fixture tty.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: tty
    steps:
      - pty:
          shell: true
          command: 'if [ -t 0 ]; then echo saw-a-tty; else echo saw-a-pipe; fi'
      - assert:
          exit_code: 0
          stdout:
            contains: saw-a-tty

When #

${atago} run tty.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: a never-matching expect fails with the pattern in the block #

skipped on Windows

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: waits forever
    steps:
      - pty:
          command: cat
          timeout: 2s
          session:
            - expect: "prompt-that-never-comes"

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 1
  • stdout contains pty expect /prompt-that-never-comes/, never appeared in the terminal transcript

Scenario: named keys transmit their documented bytes and ctrl-c aborts #

skipped on Windows

When #

# interactive (pty): cat -v

Then #

  • exit code is 0
  • stdout contains ^[[B, ^_, ^[[45;5u

Scenario: shift-tab, meta chords, and modified arrows transmit their xterm bytes #

skipped on Windows

When #

# interactive (pty): cat -v
# interactive (pty): stty raw -echo; printf "READY\r\n"; head -c 3 | cat -v
# interactive (pty): cat -v
# interactive (pty): cat -v
# interactive (pty): trap 'exit 130' INT; echo waiting; while true; do sleep 0.1; done

Then #

  • after interactive (pty): cat -v:
    • exit code is 0
    • stdout contains ^[[Z, ^[b, ^[[1;5D, ^[[1;2A, ^[[2~
  • after interactive (pty): stty raw -echo; printf "READY\r\n"; head -c 3 | cat -v:
    • exit code is 0
    • stdout contains ^[^?X
  • after interactive (pty): cat -v:
    • exit code is 0
    • stdout contains ^[[B^[[B^[[B
  • after interactive (pty): cat -v:
    • exit code is 0
    • stdout contains ^[[Z
  • after interactive (pty): trap 'exit 130' INT; echo waiting; while true; do sleep 0.1; done:
    • exit code is 130

Scenario: a bracketed paste is delivered wrapped, and refused when unasked for #

skipped on Windows

Given #

  • Fixture file unasked.atago.yaml is created.

Inputs #

Fixture unasked.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: pastes at a program that never asked
    steps:
      - pty:
          command: cat
          timeout: 5s
          session:
            - send: {paste: "ab"}

When #

# interactive (pty): stty raw -echo; printf "\033[?2004hREADY\r\n"; head -c 14 | cat -v
${atago} run unasked.atago.yaml

Then #

  • after interactive (pty): stty raw -echo; printf "\033[?2004hREADY\r\n"; head -c 14 | cat -v:
    • exit code is 0
    • stdout contains ^[[200~ab^[[201~
  • after ${atago} run unasked.atago.yaml:
    • exit code is 4
    • stdout contains has not enabled bracketed paste, ESC [?2004h

Scenario: a resize delivers the new size and the screen follows it #

skipped on Windows

When #

# interactive (pty): trap "stty size; exit 0" WINCH; stty size; while :; do sleep 0.05; done
# interactive (pty): printf 'abcdefghijKL
'; sleep 0.2

Then #

  • after interactive (pty): trap "stty size; exit 0" WINCH; stty size; while :; do sleep 0.05; done:
    • exit code is 0
    • stdout contains 40 100
  • after interactive (pty): printf 'abcdefghijKL '; sleep 0.2:
    • rendered screen contains "abcdefghij\nKL"

Scenario: a session can change the world and watch the program notice #

skipped on Windows

Given #

  • Fixture file badexec.atago.yaml is created.

Inputs #

Fixture badexec.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: the helper fails
    steps:
      - pty:
          shell: true
          command: 'sleep 5'
          timeout: 10s
          session:
            - exec: {command: "exit 7", shell: true}
            - expect: never-comes

When #

# interactive (pty): while :; do if [ -f marker.txt ]; then echo FOUND; break; fi; sleep 0.05; done
${atago} run badexec.atago.yaml

Then #

  • after interactive (pty): while :; do if [ -f marker.txt ]; then echo FOUND; break; fi; sleep 0.05; done:
    • exit code is 0
    • stdout contains FOUND
    • file marker.txt exists
  • after ${atago} run badexec.atago.yaml:
    • exit code is 4
    • stdout contains exited 7, the change the session waits for was not made

Generated artifacts #

  • marker.txt

Scenario: a mouse click is delivered as an SGR report, and refused when unasked for #

skipped on Windows

Given #

  • Fixture file unasked_mouse.atago.yaml is created.

Inputs #

Fixture unasked_mouse.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: clicks at a program that never asked
    steps:
      - pty:
          command: cat
          timeout: 5s
          session:
            - send: {mouse: {row: 1, col: 1}}

When #

# interactive (pty): stty raw -echo; printf "\033[?1002h\033[?1006hREADY\r\n"; head -c 20 | cat -v
${atago} run unasked_mouse.atago.yaml

Then #

  • after interactive (pty): stty raw -echo; printf "\033[?1002h\033[?1006hREADY\r\n"; head -c 20 | cat -v:
    • exit code is 0
    • stdout contains ^[[<0;12;5M^[[<0;12;5m
  • after ${atago} run unasked_mouse.atago.yaml:
    • exit code is 4
    • stdout contains has not enabled mouse reporting, ESC [?1000h

Scenario: screen attrs check colors and styling, not only text #

skipped on Windows

Given #

  • Fixture file badattrs.atago.yaml is created.
  • Fixture file badflag.atago.yaml is created.

Inputs #

Fixture badattrs.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: the error is not green
    steps:
      - pty:
          shell: true
          command: "printf '\\033[31mERROR\\033[0m\\r\\n'"
          rows: 4
          cols: 20
      - assert:
          screen:
            attrs:
              - {text: ERROR, fg: green}

Fixture badflag.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: plain text is not italic
    steps:
      - pty:
          shell: true
          command: "printf 'PLAIN\\r\\n'"
          rows: 4
          cols: 20
      - assert:
          screen:
            attrs:
              - {text: PLAIN, italic: true}

When #

# interactive (pty): printf '\033[1;31mERROR\033[0m plain\r\n\033[7mSELECTED\033[0m\r\n'
# interactive (pty): printf '\033[3mITAL\033[0m \033[4mUNDER\033[0m \033[5mBLINK\033[0m \033[41mONRED\033[0m\r\n'
${atago} run badattrs.atago.yaml
${atago} run badflag.atago.yaml

Then #

  • after interactive (pty): printf '\033[1;31mERROR\033[0m plain\r\n\033[7mSELECTED\033[0m\r\n':
    • rendered screen contains ERROR and shows “ERROR” in bold red and shows “plain” in not bold default and shows “SELECTED” in reverse on row 2
  • after interactive (pty): printf '\033[3mITAL\033[0m \033[4mUNDER\033[0m \033[5mBLINK\033[0m \033[41mONRED\033[0m\r\n':
    • rendered screen is checked and shows “ITAL” in italic and shows “UNDER” in underlined and shows “BLINK” in blinking and shows “ONRED” in on red and shows “ITAL” in not underlined not blinking on default
  • after ${atago} run badattrs.atago.yaml:
    • exit code is 1
    • stdout contains fg=red (wanted green)
  • after ${atago} run badflag.atago.yaml:
    • exit code is 1
    • stdout contains italic

Scenario: an unknown key name is a load-time error listing the vocabulary #

Given #

  • Fixture file badkey.atago.yaml is created.

Inputs #

Fixture badkey.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: typo'd key
    steps:
      - pty:
          command: cat
          session:
            - send: { key: entr }

When #

${atago} run badkey.atago.yaml

Then #

  • exit code is 2
  • stderr contains not a supported key (supported: enter, tab

Scenario: screen asserts see the final frame where the transcript sees history #

skipped on Windows

When #

# interactive (pty): printf 'loading...
'; printf 'done.      
'

Then #

  • rendered screen line 1 equals an exact value
  • rendered screen does not contain loading
  • stdout contains loading

Scenario: a wide character at the right margin autowraps #

skipped on Windows

When #

# interactive (pty): printf '日本X'; sleep 0.2

Then #

  • rendered screen line 1 equals an exact value
  • rendered screen line 2 equals an exact value

Scenario: a wide character that no longer fits wraps instead of vanishing #

skipped on Windows

When #

# interactive (pty): printf '日本語'; sleep 0.2

Then #

  • rendered screen line 1 equals an exact value
  • rendered screen line 2 equals an exact value

Scenario: screen preserves a decomposed grapheme’s combining mark #

skipped on Windows

When #

# interactive (pty): printf '\303\251 e\314\201'; sleep 0.2

Then #

  • rendered screen line 1 equals an exact value
  • rendered screen contains é

Scenario: a screen snapshot round-trips through update and compare #

skipped on Windows

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: menu frame
    steps:
      - pty:
          shell: true
          command: "printf '\\033[2J\\033[HMain Menu\\r\\n> Settings\\r\\n'"
          rows: 10
          cols: 40
      - assert:
          screen:
            snapshot: snapshots/menu.txt

When #

${atago} run --update-snapshots inner.atago.yaml
${atago} run inner.atago.yaml

Then #

  • after ${atago} run --update-snapshots inner.atago.yaml:
    • exit code is 0
    • file snapshots/menu.txt contains > Settings
  • after ${atago} run inner.atago.yaml:
    • exit code is 0

Scenario: a screen assert without a pty step is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: no pty
    steps:
      - run: {command: echo hi}
      - assert:
          screen: {contains: hi}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains requires a preceding pty step

Scenario: a send referencing an undefined variable is an execution error, not typed literally #

skipped on Windows

Given #

  • Fixture file typo.atago.yaml is created.

Inputs #

Fixture typo.atago.yaml:

version: "1"
suite:
  name: typo
scenarios:
  - name: typo in a send variable
    steps:
      - pty:
          command: cat
          timeout: 5s
          session:
            - send: "${no_such_var}\n"

When #

${atago} run typo.atago.yaml

Then #

  • exit code is 4
  • stdout contains no variable with that name is defined, $${no_such_var}

Scenario: an expect does not match the echo of its own send #

skipped on Windows

Given #

  • Fixture file echo.atago.yaml is created.

Inputs #

Fixture echo.atago.yaml:

version: "1"
suite:
  name: echo
scenarios:
  - name: the program never produced this
    steps:
      - pty:
          command: "sh -c 'echo ready; sleep 30'"
          timeout: 4s
          session:
            - expect: "ready"
            - send: "NEVER-PRODUCED-BY-THE-PROGRAM\n"
            - expect: "NEVER-PRODUCED-BY-THE-PROGRAM"

When #

${atago} run echo.atago.yaml

Then #

  • exit code is 1
  • stdout contains pty expect /NEVER-PRODUCED-BY-THE-PROGRAM/, never appeared in the terminal transcript

Scenario: a program’s own copy of the input still satisfies an expect #

skipped on Windows

When #

# interactive (pty): cat

Then #

  • exit code is 0

Scenario: a signaled child reports 128+signal from both runners #

skipped on Windows

When #

# interactive (pty): sh -c 'kill -TERM $$'
kill -TERM $$
# interactive (pty): sh -c 'kill -INT $$'

Then #

  • after interactive (pty): sh -c 'kill -TERM $$':
    • exit code is 143
  • after kill -TERM $$:
    • exit code is 143
  • after interactive (pty): sh -c 'kill -INT $$':
    • exit code is 130

atago self-hosting / pty (portable) #

Source: test/e2e/atago/pty_portable.atago.yaml

Scenario: a pty step starts a command, captures its output, and reports exit 0 #

When #

# interactive (pty): echo hello from a pty

Then #

  • exit code is 0
  • stdout contains hello from a pty

Scenario: a pty step surfaces a command’s non-zero exit code #

When #

# interactive (pty): echo bye && exit 3

Then #

  • exit code is 3

Scenario: sequential expects match successive output in declaration order #

When #

# interactive (pty): echo first && echo second && echo third

Then #

  • exit code is 0
  • stdout contains first, third

Scenario: an expect pattern is a regular expression, not a literal #

When #

# interactive (pty): echo item-42-done

Then #

  • exit code is 0

Scenario: a screen assert reads the rendered frame sized by rows and cols #

When #

# interactive (pty): echo rendered line

Then #

  • exit code is 0
  • rendered screen contains rendered line

Scenario: a mid-session resize succeeds on every platform #

When #

# interactive (pty): echo before resize && echo after resize

Then #

  • exit code is 0
  • stdout contains before resize, after resize
  • rendered screen contains after resize

Scenario: a pty step drives the atago binary directly with no shell #

When #

# interactive (pty): ${atago} version

Then #

  • exit code is 0
  • stdout contains atago

Scenario: a pty drives atago running an inner spec to a green result #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: echo
    steps:
      - run:
          shell: true
          command: echo inner-ok
      - assert:
          exit_code: 0
          stdout:
            contains: inner-ok

When #

# interactive (pty): ${atago} run inner.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: a never-matching expect fails and names the pattern in the transcript #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: waits for output that never comes
    steps:
      - pty:
          shell: true
          command: echo present
          timeout: 2s
          session:
            - expect: "absent-forever"

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 1
  • stdout contains pty expect /absent-forever/, never appeared in the terminal transcript

Scenario: a stable_for above the session budget is a load-time error #

Given #

  • Fixture file stable.atago.yaml is created.

Inputs #

Fixture stable.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: waits longer than the session allows
    steps:
      - pty:
          shell: true
          command: echo hi
          timeout: 2s
          session:
            - expect_screen: {contains: hi, stable_for: 60s}

When #

${atago} run stable.atago.yaml

Then #

  • exit code is 2
  • stderr contains must not exceed

atago self-hosting / record (spec skeleton from an observed run) #

Source: test/e2e/atago/record.atago.yaml

Scenario: record then run round-trips green #

When #

${atago} record --out recorded.atago.yaml -- ${atago} version
${atago} run recorded.atago.yaml

Then #

  • after ${atago} record --out recorded.atago.yaml -- ${atago} version:
    • exit code is 0
    • stderr contains wrote recorded.atago.yaml
    • file recorded.atago.yaml contains contains: atago
  • after ${atago} run recorded.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: refusing to overwrite without –force #

Given #

  • Fixture file existing.atago.yaml is created.

Inputs #

Fixture existing.atago.yaml:

precious

When #

${atago} record --out existing.atago.yaml -- ${atago} version
${atago} record --force --out existing.atago.yaml -- ${atago} version

Then #

  • after ${atago} record --out existing.atago.yaml -- ${atago} version:
    • exit code is 3
    • stderr contains use --force to overwrite
    • file existing.atago.yaml contains precious
  • after ${atago} record --force --out existing.atago.yaml -- ${atago} version:
    • exit code is 0
    • file existing.atago.yaml contains exit_code: 0

Scenario: record –pty refuses an existing –out before driving the session #

Given #

  • Fixture file taken.atago.yaml is created.

Inputs #

Fixture taken.atago.yaml:

precious

When #

${atago} record --pty --out taken.atago.yaml -- echo hi

Then #

  • exit code is 3
  • stderr contains use --force to overwrite
  • file taken.atago.yaml contains precious

Scenario: an observed stderr diagnostic is anchored, not dropped #

skipped on Windows

When #

${atago} record --shell --out diag.atago.yaml -- 'echo "error: unknown flag" >&2; exit 2'
${atago} run diag.atago.yaml

Then #

  • after ${atago} record --shell --out diag.atago.yaml -- 'echo "error: unknown flag" >&2; exit 2':
    • exit code is 0
    • file diag.atago.yaml contains exit_code: 2, stderr:, error: unknown flag
    • file diag.atago.yaml does not contain empty: true
  • after ${atago} run diag.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: created files become exists asserts (shell mode) #

skipped on Windows

When #

${atago} record --shell --out gen.atago.yaml -- 'echo made > out.txt; echo done'
${atago} run gen.atago.yaml

Then #

  • after ${atago} record --shell --out gen.atago.yaml -- 'echo made > out.txt; echo done':
    • exit code is 0
    • file gen.atago.yaml contains path: out.txt, exists: true
    • file gen.atago.yaml contains shell: true
  • after ${atago} run gen.atago.yaml:
    • exit code is 0

Scenario: snapshot mode writes a golden the run then matches #

skipped on Windows

When #

${atago} record --snapshot --out snapdemo.atago.yaml -- echo stable output
${atago} run snapdemo.atago.yaml

Then #

  • after ${atago} record --snapshot --out snapdemo.atago.yaml -- echo stable output:
    • exit code is 0
    • file snapdemo.atago.yaml contains snapshot: snapshots/snapdemo.stdout.txt
    • file snapshots/snapdemo.stdout.txt contains stable output
  • after ${atago} run snapdemo.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: no command is a usage error #

When #

${atago} record

Then #

  • exit code is 3
  • stderr contains no command given

Scenario: argv boundaries survive spaced arguments #

skipped on Windows

When #

${atago} record --out spaced.atago.yaml -- printf %s 'hello world'
${atago} run spaced.atago.yaml

Then #

  • after ${atago} record --out spaced.atago.yaml -- printf %s 'hello world':
    • exit code is 0
    • file spaced.atago.yaml contains contains: hello world
  • after ${atago} run spaced.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: a shell metacharacter argument stays one token #

skipped on Windows

When #

${atago} record --out meta.atago.yaml -- printf %s "foo|bar"
${atago} run meta.atago.yaml

Then #

  • after ${atago} record --out meta.atago.yaml -- printf %s "foo|bar":
    • exit code is 0
    • file meta.atago.yaml contains contains: foo|bar
  • after ${atago} run meta.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: record –pty records a live session and the generated spec replays green #

skipped on Windows

When #

# interactive (pty): ${atago} record --pty --out generated.atago.yaml -- sh -c 'printf PROMPT; read n; echo hi-$n'
${atago} run generated.atago.yaml

Then #

  • after interactive (pty): ${atago} record --pty --out generated.atago.yaml -- sh -c 'printf PROMPT; read n; echo hi-$n':
    • exit code is 0
    • file generated.atago.yaml contains - pty:, - send:
  • after ${atago} run generated.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: record –pty of a silent program anchors on nothing rather than on the echo #

skipped on Windows

When #

# interactive (pty): ${atago} record --pty --out silent.atago.yaml -- sh -c 'read a; read b; echo done'
${atago} run silent.atago.yaml

Then #

  • after interactive (pty): ${atago} record --pty --out silent.atago.yaml -- sh -c 'read a; read b; echo done':
    • exit code is 0
    • file silent.atago.yaml does not contain - expect:
  • after ${atago} run silent.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: record –pty of a no-input command yields a session-less spec that replays green #

skipped on Windows

When #

# interactive (pty): ${atago} record --pty --out echo.atago.yaml -- echo done
${atago} run echo.atago.yaml

Then #

  • after interactive (pty): ${atago} record --pty --out echo.atago.yaml -- echo done:
    • exit code is 0
    • file echo.atago.yaml contains - pty:, command: echo done
    • file echo.atago.yaml does not contain - send:, session:
  • after ${atago} run echo.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: a prompt with regex metacharacters is escaped in the generated expect #

skipped on Windows

When #

# interactive (pty): ${atago} record --pty --out meta.atago.yaml -- sh -c 'printf "Continue? (y/n): "; read a; echo got-$a'
${atago} run meta.atago.yaml

Then #

  • after interactive (pty): ${atago} record --pty --out meta.atago.yaml -- sh -c 'printf "Continue? (y/n): "; read a; echo got-$a':
    • exit code is 0
    • file meta.atago.yaml contains expect: "Continue\\? \\(y/n\\):", - send:
  • after ${atago} run meta.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: recorded text containing dollar-brace round-trips as literal text #

skipped on Windows

When #

${atago} record --out dollar.atago.yaml -- printf %s 'literal $${HOME} here'
${atago} run dollar.atago.yaml

Then #

  • after ${atago} record --out dollar.atago.yaml -- printf %s 'literal $${HOME} here':
    • exit code is 0
    • file dollar.atago.yaml contains $$${HOME}
  • after ${atago} run dollar.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: a recorded secret placeholder replays green with the env set and is guarded when unset #

skipped on Windows

Given #

  • Environment variables are set: ATAGO_SECRET_1.

When #

# interactive (pty): ${atago} record --pty --out sec.atago.yaml -- sh -c 'stty -echo; printf "Password: "; read pw; stty echo; printf "\naccepted\n"'
${atago} run sec.atago.yaml
${atago} run sec.atago.yaml

Then #

  • after interactive (pty): ${atago} record --pty --out sec.atago.yaml -- sh -c 'stty -echo; printf "Password: "; read pw; stty echo; printf "\naccepted\n"':
    • exit code is 0
    • file sec.atago.yaml contains ${env:ATAGO_SECRET_1}
    • file sec.atago.yaml does not contain hunter2
  • after ${atago} run sec.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed
  • after ${atago} run sec.atago.yaml:
    • exit code is 4
    • stdout contains ATAGO_SECRET_1 is not set

Scenario: a raw-mode (TUI) keystroke is recorded literally, not as a secret #

skipped on Windows

When #

# interactive (pty): ${atago} record --pty --out tui.atago.yaml -- sh -c 'stty -icanon -echo min 1 time 0; printf READY; head -c 1 >/dev/null; stty sane; echo BYE'
${atago} run tui.atago.yaml

Then #

  • after interactive (pty): ${atago} record --pty --out tui.atago.yaml -- sh -c 'stty -icanon -echo min 1 time 0; printf READY; head -c 1 >/dev/null; stty sane; echo BYE':
    • exit code is 0
    • file tui.atago.yaml contains - send: "j"
    • file tui.atago.yaml does not contain ATAGO_SECRET
  • after ${atago} run tui.atago.yaml:
    • exit code is 0
    • stdout contains 1 passed

Scenario: record –pty of a never-exiting program times out instead of hanging #

skipped on Windows

When #

${atago} record --pty --timeout 2s --out wedged.atago.yaml -- tail -f /dev/null

Then #

  • exit code is 4
  • stderr contains did not exit within 2s
  • stderr contains use --timeout to adjust
  • file wedged.atago.yaml contains - pty:

atago self-hosting / report formats agree on outcomes #

Source: test/e2e/atago/report_formats.atago.yaml

Scenario: json report carries per-scenario verdicts and a failures array #

Given #

  • Fixture file mixed.atago.yaml is created.

Inputs #

Fixture mixed.atago.yaml:

version: "1"
suite:
  name: mixed
scenarios:
  - name: alpha passes
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}
  - name: beta passes
    steps:
      - run: {shell: true, command: "echo hi"}
      - assert: {stdout: {contains: hi}}
  - name: gamma fails
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 9}
  - name: delta is skipped
    skip: {env: PATH}
    steps:
      - run: {shell: true, command: "exit 0"}

When #

${atago} run --ci --report json mixed.atago.yaml

Then #

  • exit code is 1
  • stdout at $.suites[0].status equals failed; at $.suites[0].scenarios[0].status equals passed; at $.suites[0].scenarios[2].status equals failed; at $.suites[0].scenarios[3].status equals skipped; at $.suites[0].failures[0].scenario equals gamma fails

Scenario: junit report tallies tests, failures, skipped, and errors #

Given #

  • Fixture file mixed.atago.yaml is created.

Inputs #

Fixture mixed.atago.yaml:

version: "1"
suite:
  name: mixed
scenarios:
  - name: alpha passes
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}
  - name: beta passes
    steps:
      - run: {shell: true, command: "echo hi"}
      - assert: {stdout: {contains: hi}}
  - name: gamma fails
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 9}
  - name: delta is skipped
    skip: {env: PATH}
    steps:
      - run: {shell: true, command: "exit 0"}

When #

${atago} run --ci --report junit mixed.atago.yaml

Then #

  • exit code is 1
  • stdout contains tests="4", failures="1", errors="0", skipped="1", <skipped

Scenario: tap report emits the plan, a not ok line, and a SKIP directive #

Given #

  • Fixture file mixed.atago.yaml is created.

Inputs #

Fixture mixed.atago.yaml:

version: "1"
suite:
  name: mixed
scenarios:
  - name: alpha passes
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}
  - name: beta passes
    steps:
      - run: {shell: true, command: "echo hi"}
      - assert: {stdout: {contains: hi}}
  - name: gamma fails
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 9}
  - name: delta is skipped
    skip: {env: PATH}
    steps:
      - run: {shell: true, command: "exit 0"}

When #

${atago} run --ci --report tap mixed.atago.yaml

Then #

  • exit code is 1
  • stdout contains TAP version 13, 1..4, not ok 3 - mixed / gamma fails, # SKIP

Scenario: gha report annotates the failure and summarizes the counts #

Given #

  • Fixture file mixed.atago.yaml is created.

Inputs #

Fixture mixed.atago.yaml:

version: "1"
suite:
  name: mixed
scenarios:
  - name: alpha passes
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}
  - name: beta passes
    steps:
      - run: {shell: true, command: "echo hi"}
      - assert: {stdout: {contains: hi}}
  - name: gamma fails
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 9}
  - name: delta is skipped
    skip: {env: PATH}
    steps:
      - run: {shell: true, command: "exit 0"}

When #

${atago} run --ci --report gha mixed.atago.yaml

Then #

  • exit code is 1
  • stdout contains ::error title=mixed / gamma fails::, ::notice title=atago::4 scenarios: 2 passed, 1 failed, 0 errored, 1 skipped

Scenario: console report prints the same counts in its summary line #

Given #

  • Fixture file mixed.atago.yaml is created.

Inputs #

Fixture mixed.atago.yaml:

version: "1"
suite:
  name: mixed
scenarios:
  - name: alpha passes
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}
  - name: beta passes
    steps:
      - run: {shell: true, command: "echo hi"}
      - assert: {stdout: {contains: hi}}
  - name: gamma fails
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 9}
  - name: delta is skipped
    skip: {env: PATH}
    steps:
      - run: {shell: true, command: "exit 0"}

When #

${atago} run --ci --report console mixed.atago.yaml

Then #

  • exit code is 1
  • stdout contains 4 scenarios: 2 passed, 1 failed, 0 errored, 1 skipped

Scenario: an all-passing run reports a zero-failure suite and exits zero #

Given #

  • Fixture file allpass.atago.yaml is created.

Inputs #

Fixture allpass.atago.yaml:

version: "1"
suite:
  name: allpass
scenarios:
  - name: only passes
    steps:
      - run: {shell: true, command: "echo ok"}
      - assert: {stdout: {contains: ok}}

When #

${atago} run --ci --report json allpass.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites[0].status equals passed; at $.suites[0].scenarios[0].status equals passed

Scenario: a spec that failed to load is named by every report format #

Given #

  • Fixture file good.atago.yaml is created.
  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture good.atago.yaml:

version: "1"
suite:
  name: good
scenarios:
  - name: passes
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}

Fixture bad.atago.yaml:

version: "1"
suite: {name: broken}
scenarios: [ this is not a scenario

When #

${atago} run --report json .
${atago} run --report junit .
${atago} run --report tap .
${atago} run --report gha .

Then #

  • after ${atago} run --report json .:
    • exit code is 2
    • stdout at $.load_failures[0].spec_path equals bad.atago.yaml; at $.suites[0].status equals passed
  • after ${atago} run --report junit .:
    • exit code is 2
    • stdout contains errors="1", bad.atago.yaml
  • after ${atago} run --report tap .:
    • exit code is 2
    • stdout contains 1..2, not ok 1 - bad.atago.yaml
  • after ${atago} run --report gha .:
    • exit code is 2
    • stdout contains ::error file=bad.atago.yaml,title=bad.atago.yaml, 1 spec failed to load

Scenario: a run whose specs all failed to load still reports them #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite: {name: broken}
scenarios: [ this is not a scenario

When #

${atago} run --report json .
${atago} run --report tap .

Then #

  • after ${atago} run --report json .:
    • exit code is 2
    • stdout at $.load_failures[0].spec_path equals bad.atago.yaml
  • after ${atago} run --report tap .:
    • exit code is 2
    • stdout contains 1..1, not ok 1 - bad.atago.yaml

Scenario: an errored step is counted as an error, not a failure, across formats #

Given #

  • Fixture file errored.atago.yaml is created.

Inputs #

Fixture errored.atago.yaml:

version: "1"
suite:
  name: errored
scenarios:
  - name: missing stdin file errors
    steps:
      - run: {shell: true, command: "cat", stdin: {file: "nope.txt"}}

When #

${atago} run --ci --report junit errored.atago.yaml

Then #

  • exit code is 4
  • stdout contains errors="1", failures="0", <error

Scenario: a failed teardown surfaces in junit, tap, and gha without changing the verdict #

Given #

  • Fixture file td.atago.yaml is created.

Inputs #

Fixture td.atago.yaml:

version: "1"
suite:
  name: td
scenarios:
  - name: passes but cleanup fails
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}
    teardown:
      - run: {shell: true, command: "exit 7"}
      - assert: {exit_code: 0}

When #

${atago} run --ci --report junit td.atago.yaml
${atago} run --ci --report tap td.atago.yaml
${atago} run --ci --report gha td.atago.yaml

Then #

  • after ${atago} run --ci --report junit td.atago.yaml:
    • exit code is 0
    • stdout contains failures="0", <system-err>teardown failed, exit code 7
  • after ${atago} run --ci --report tap td.atago.yaml:
    • exit code is 0
    • stdout contains ok 1 - td / passes but cleanup fails, # teardown failed: assert exit_code is 0
  • after ${atago} run --ci --report gha td.atago.yaml:
    • exit code is 0
    • stdout contains ::warning title=td / passes but cleanup fails::teardown failed, does not contain ::error

Scenario: a snapshot rewrite is reported by the console and every machine format #

Given #

  • Fixture file snap.atago.yaml is created.

Inputs #

Fixture snap.atago.yaml:

version: "1"
suite:
  name: snap
scenarios:
  - name: greets
    steps:
      - run: {shell: true, command: "echo greetings"}
      - assert: {stdout: {snapshot: greet.snap}}

When #

${atago} run --ci --update-snapshots snap.atago.yaml
${atago} run --ci --update-snapshots --report json snap.atago.yaml
${atago} run --ci --update-snapshots --report gha snap.atago.yaml
${atago} run --ci --update-snapshots --report tap snap.atago.yaml
${atago} run --ci snap.atago.yaml

Then #

  • after ${atago} run --ci --update-snapshots snap.atago.yaml:
    • exit code is 0
    • stdout contains 1 snapshot updated
  • after ${atago} run --ci --update-snapshots --report json snap.atago.yaml:
    • exit code is 0
    • stdout at $.snapshots_updated equals 1
  • after ${atago} run --ci --update-snapshots --report gha snap.atago.yaml:
    • exit code is 0
    • stdout contains ::warning title=atago::1 snapshot updated
  • after ${atago} run --ci --update-snapshots --report tap snap.atago.yaml:
    • exit code is 0
    • stdout contains # 1 snapshot updated
  • after ${atago} run --ci snap.atago.yaml:
    • exit code is 0
    • stdout does not contain snapshot updated

atago self-hosting / reports #

Source: test/e2e/atago/reports.atago.yaml

Scenario: JUnit report is XML with a testsuite and testcase #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: echo
    steps:
      - run:
          command: echo hi
      - assert:
          stdout:
            contains: hi

When #

${atago} run --report junit ok.atago.yaml

Then #

  • exit code is 0
  • stdout contains <testsuites, <testcase

Scenario: GitHub Actions annotations are emitted on failure #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: nope
    steps:
      - run:
          command: echo Bob
      - assert:
          stdout:
            contains: Alice

When #

${atago} run --report gha bad.atago.yaml

Then #

  • exit code is 1
  • stdout contains ::error

Scenario: TAP report is a numbered TAP 13 stream with ok / not ok points #

Given #

  • Fixture file mixed.atago.yaml is created.

Inputs #

Fixture mixed.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: good
    steps:
      - run:
          command: echo hi
      - assert:
          stdout:
            contains: hi
  - name: bad
    steps:
      - run:
          command: echo Bob
      - assert:
          stdout:
            contains: Alice

When #

${atago} run --report tap mixed.atago.yaml

Then #

  • exit code is 1
  • stdout line 1 equals an exact value
  • stdout line 2 equals an exact value
  • stdout contains ok 1 - sample / good, not ok 2 - sample / bad

Scenario: failure artifacts are written and referenced in the JSON report #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: nope
    steps:
      - run:
          command: echo Bob
      - assert:
          stdout:
            contains: Alice

When #

${atago} run --report json --artifacts-dir arts bad.atago.yaml
cat arts/*/*/step-*-stdout.actual.txt

Then #

  • after ${atago} run --report json --artifacts-dir arts bad.atago.yaml:
    • exit code is 1
    • stdout contains "artifacts", .actual.txt
  • after cat arts/*/*/step-*-stdout.actual.txt:
    • exit code is 0
    • stdout contains Bob

Scenario: a multi-line snapshot failure renders a unified diff with hunks #

Given #

  • Fixture file diffspec.atago.yaml is created.
  • Fixture file diffspec.atago.yaml is created.

Inputs #

Fixture diffspec.atago.yaml:

version: "1"
suite:
  name: diffdemo
scenarios:
  - name: multi-line output matches its golden
    steps:
      - run:
          shell: true
          command: "printf 'alpha\\nbeta\\ngamma\\n'"
      - assert:
          stdout:
            snapshot: snaps/out.txt

Fixture diffspec.atago.yaml:

version: "1"
suite:
  name: diffdemo
scenarios:
  - name: multi-line output matches its golden
    steps:
      - run:
          shell: true
          command: "printf 'alpha\\nBETA\\ngamma\\n'"
      - assert:
          stdout:
            snapshot: snaps/out.txt

When #

${atago} run --update-snapshots diffspec.atago.yaml
${atago} run diffspec.atago.yaml
${atago} run --report json diffspec.atago.yaml

Then #

  • after ${atago} run --update-snapshots diffspec.atago.yaml:
    • exit code is 0
  • after ${atago} run diffspec.atago.yaml:
    • exit code is 1
    • stdout contains Diff (-expected +actual):, --- snapshot (golden), -beta, +BETA, snaps/out.txt
  • after ${atago} run --report json diffspec.atago.yaml:
    • exit code is 1
    • stdout contains "diff":

Scenario: a failure names the command it was observed under #

Given #

  • Fixture file attrib.atago.yaml is created.

Inputs #

Fixture attrib.atago.yaml:

version: "1"
suite: {name: attrib}
scenarios:
  - name: the first command fails and a later one passes
    steps:
      - run: {shell: true, command: "printf 'early\n'; exit 3"}
      - assert: {exit_code: 0}
      - run: {shell: true, command: "printf 'later-and-fine\n'"}
      - assert: {exit_code: 0}

When #

${atago} run attrib.atago.yaml
${atago} run --report json attrib.atago.yaml

Then #

  • after ${atago} run attrib.atago.yaml:
    • exit code is 1
    • stdout contains printf 'early
    • stdout does not contain later-and-fine
  • after ${atago} run --report json attrib.atago.yaml:
    • exit code is 1
    • stdout at $.suites[0].failures[0].command matches /early/

Scenario: a failure against an empty stream reports the stream as empty #

Given #

  • Fixture file silent.atago.yaml is created.

Inputs #

Fixture silent.atago.yaml:

version: "1"
suite: {name: silent}
scenarios:
  - name: a command that prints nothing is asserted to have printed
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {stdout: {contains: "PASSED"}}

When #

${atago} run silent.atago.yaml

Then #

  • exit code is 1
  • stdout matches /Actual:\n \(empty\)/

Scenario: an exit_code failure states that the command printed nothing #

Given #

  • Fixture file quiet.atago.yaml is created.

Inputs #

Fixture quiet.atago.yaml:

version: "1"
suite: {name: quiet}
scenarios:
  - name: exit code failure with no output at all
    steps:
      - run: {shell: true, command: "exit 7"}
      - assert: {exit_code: 0}

When #

${atago} run quiet.atago.yaml

Then #

  • exit code is 1
  • stdout matches /Stdout/Stderr:\n \(empty\)/

Scenario: a stdout failure points at stderr when the text is there #

Given #

  • Fixture file wrongstream.atago.yaml is created.

Inputs #

Fixture wrongstream.atago.yaml:

version: "1"
suite: {name: wrongstream}
scenarios:
  - name: assert stdout when the tool prints to stderr
    steps:
      - run: {shell: true, command: "printf 'boom: no such file\n' 1>&2; exit 1"}
      - assert: {stdout: {contains: "boom"}}

When #

${atago} run wrongstream.atago.yaml

Then #

  • exit code is 1
  • stdout contains stderr satisfies this assertion (assert `stderr:` instead?)

Scenario: an empty stream that ended early says so in the failure block #

skipped on Windows

Given #

  • Fixture file earlyeof.atago.yaml is created.

Inputs #

Fixture earlyeof.atago.yaml:

version: "1"
suite: {name: earlyeof}
scenarios:
  - name: a command that closes stdout then keeps running
    steps:
      - run: {shell: true, command: "exec 1>&-; sleep 1; exit 0"}
      - assert:
          exit_code: 0
          stdout: {contains: "anything"}

When #

${atago} run earlyeof.atago.yaml

Then #

  • exit code is 1
  • stdout contains before the command exited, never connected to atago's pipe

atago self-hosting / rerun-failed #

Source: test/e2e/atago/rerun.atago.yaml

Scenario: a failing run is recorded and rerun-failed selects only it #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: always-green
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}
  - name: always-red
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

When #

${atago} run inner.atago.yaml
${atago} run --rerun-failed inner.atago.yaml

Then #

  • after ${atago} run inner.atago.yaml:
    • exit code is 1
    • file .atago/last-failed.json exists
    • file .atago/last-failed.json at $.failed[0].scenario equals always-red
  • after ${atago} run --rerun-failed inner.atago.yaml:
    • exit code is 1
    • stdout contains always-red

Generated artifacts #

  • .atago/last-failed.json

Scenario: rerun-failed with nothing recorded is a no-op success #

Given #

  • Fixture file green.atago.yaml is created.

Inputs #

Fixture green.atago.yaml:

version: "1"
suite:
  name: green
scenarios:
  - name: passes
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}

When #

${atago} run --rerun-failed green.atago.yaml

Then #

  • exit code is 0
  • stderr contains nothing to rerun

Scenario: rerun-failed with a filter preserves the still-failing scenarios it did not run #

Given #

  • Fixture file two.atago.yaml is created.

Inputs #

Fixture two.atago.yaml:

version: "1"
suite:
  name: two
scenarios:
  - name: red-a
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}
  - name: red-b
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

When #

${atago} run two.atago.yaml
${atago} run --rerun-failed --filter red-a two.atago.yaml

Then #

  • after ${atago} run two.atago.yaml:
    • exit code is 1
  • after ${atago} run --rerun-failed --filter red-a two.atago.yaml:
    • exit code is 1
    • file .atago/last-failed.json contains red-a, red-b

Scenario: a fail-fast run keeps the recorded failures it never got to #

Given #

  • Fixture file two.atago.yaml is created.
  • Fixture file two.atago.yaml is created.

Inputs #

Fixture two.atago.yaml:

version: "1"
suite:
  name: two
scenarios:
  - name: red-a
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}
  - name: red-b
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

Fixture two.atago.yaml:

version: "1"
suite:
  name: two
scenarios:
  - name: red-a
    steps:
      - run: {shell: true, command: "exit 0"}
      - assert: {exit_code: 0}
  - name: red-b
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

When #

${atago} run two.atago.yaml
${atago} run --fail-fast --parallel 1 two.atago.yaml
${atago} run --rerun-failed two.atago.yaml

Then #

  • after ${atago} run two.atago.yaml:
    • exit code is 1
  • after ${atago} run --fail-fast --parallel 1 two.atago.yaml:
    • exit code is 1
    • stdout contains 1 skipped
    • file .atago/last-failed.json contains red-a, red-b
  • after ${atago} run --rerun-failed two.atago.yaml:
    • exit code is 1
    • stdout contains red-b

Scenario: a fail-fast run does not blame a rename for the spec it never loaded #

Given #

  • Fixture file a.atago.yaml is created.
  • Fixture file b.atago.yaml is created.

Inputs #

Fixture a.atago.yaml:

version: "1"
suite:
  name: a
scenarios:
  - name: red-a
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

Fixture b.atago.yaml:

version: "1"
suite:
  name: b
scenarios:
  - name: red-b
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

When #

${atago} run a.atago.yaml b.atago.yaml
${atago} run --rerun-failed --fail-fast --parallel 1 a.atago.yaml b.atago.yaml

Then #

  • after ${atago} run a.atago.yaml b.atago.yaml:
    • exit code is 1
  • after ${atago} run --rerun-failed --fail-fast --parallel 1 a.atago.yaml b.atago.yaml:
    • exit code is 1
    • stderr does not contain renamed or removed
    • file .atago/last-failed.json contains red-a, red-b

Scenario: rerun-failed names the recorded failures that no longer match #

Given #

  • Fixture file two.atago.yaml is created.
  • Fixture file two.atago.yaml is created.

Inputs #

Fixture two.atago.yaml:

version: "1"
suite:
  name: two
scenarios:
  - name: red-a
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}
  - name: red-b
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

Fixture two.atago.yaml:

version: "1"
suite:
  name: two
scenarios:
  - name: red-a-renamed
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}
  - name: red-b
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

When #

${atago} run two.atago.yaml
${atago} run --rerun-failed two.atago.yaml

Then #

  • after ${atago} run two.atago.yaml:
    • exit code is 1
  • after ${atago} run --rerun-failed two.atago.yaml:
    • exit code is 1
    • stderr contains 1 recorded failing scenario did not match, red-a
    • file .atago/last-failed.json contains red-a, red-b

Scenario: rerun-failed does not blame a rename for a spec it was not aimed at #

Given #

  • Fixture file a.atago.yaml is created.
  • Fixture file b.atago.yaml is created.

Inputs #

Fixture a.atago.yaml:

version: "1"
suite:
  name: a
scenarios:
  - name: red-a
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

Fixture b.atago.yaml:

version: "1"
suite:
  name: b
scenarios:
  - name: red-b
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

When #

${atago} run a.atago.yaml b.atago.yaml
${atago} run --rerun-failed a.atago.yaml

Then #

  • after ${atago} run a.atago.yaml b.atago.yaml:
    • exit code is 1
  • after ${atago} run --rerun-failed a.atago.yaml:
    • exit code is 1
    • stderr does not contain renamed or removed
    • stderr contains outside this run's targets, red-b
    • file .atago/last-failed.json contains red-a, red-b

Scenario: rerun-failed stays quiet when every recorded failure still exists #

Given #

  • Fixture file two.atago.yaml is created.

Inputs #

Fixture two.atago.yaml:

version: "1"
suite:
  name: two
scenarios:
  - name: red-a
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}
  - name: red-b
    steps:
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

When #

${atago} run two.atago.yaml
${atago} run --rerun-failed two.atago.yaml

Then #

  • after ${atago} run two.atago.yaml:
    • exit code is 1
  • after ${atago} run --rerun-failed two.atago.yaml:
    • exit code is 1
    • stderr does not contain did not match the current specs

atago self-hosting / retry until #

Source: test/e2e/atago/retry.atago.yaml

Scenario: retry polls until the condition becomes true #

Given #

  • Fixture file ready.atago.yaml is created.

Inputs #

Fixture ready.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: becomes ready on a later attempt
    steps:
      - run:
          shell: true
          command: "if [ -f marker ]; then echo ready; else touch marker; echo waiting; fi"
          retry:
            times: 5
            interval: 10ms
            until:
              stdout:
                contains: ready
      - assert:
          stdout:
            contains: ready

When #

${atago} run ready.atago.yaml

Then #

  • exit code is 0
  • stdout contains passed

Scenario: retry fails the inner spec when until never holds #

Given #

  • Fixture file never.atago.yaml is created.

Inputs #

Fixture never.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: never ready
    steps:
      - run:
          command: echo waiting
          retry:
            times: 3
            interval: 1ms
            until:
              stdout:
                contains: ready

When #

${atago} run never.atago.yaml

Then #

  • exit code is 1

Scenario: until with a changes target is a load-time error #

Given #

  • Fixture file badchanges.atago.yaml is created.

Inputs #

Fixture badchanges.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: unsatisfiable until
    steps:
      - run:
          command: echo hi
          retry:
            times: 3
            until:
              changes:
                created: [out.txt]

When #

${atago} run badchanges.atago.yaml

Then #

  • exit code is 2
  • stderr contains retry.until.changes cannot be satisfied, steps[0].run

atago self-hosting / run #

Source: test/e2e/atago/run.atago.yaml

Scenario: a passing spec exits zero and reports PASS #

Given #

  • Fixture file passing.atago.yaml is created.

Inputs #

Fixture passing.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: exit 0 succeeds
    steps:
      - run:
          shell: true
          command: "exit 0"
      - assert:
          exit_code: 0
          stderr:
            empty: true

When #

${atago} run passing.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASS
  • stderr is empty

Scenario: a failing assertion exits one and reports the failure #

Given #

  • Fixture file failing.atago.yaml is created.

Inputs #

Fixture failing.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: exit 1 should pass
    steps:
      - run:
          shell: true
          command: "exit 1"
      - assert:
          exit_code: 0

When #

${atago} run failing.atago.yaml

Then #

  • exit code is 1
  • stdout contains FAILED, expected exit code 0, (failing.atago.yaml)

Scenario: an exit_code failure surfaces the command’s stderr #

skipped on Windows

Given #

  • Fixture file stderr_cause.atago.yaml is created.

Inputs #

Fixture stderr_cause.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: converter fails loudly
    steps:
      - run:
          shell: true
          command: "echo conversion aborted: bad header 1>&2; exit 3"
      - assert:
          exit_code: 0

When #

${atago} run stderr_cause.atago.yaml

Then #

  • exit code is 1
  • stdout contains Stderr:, conversion aborted: bad header

Scenario: an exit_code failure surfaces the command’s stderr (windows) #

only on Windows

Given #

  • Fixture file stderr_cause.atago.yaml is created.

Inputs #

Fixture stderr_cause.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: converter fails loudly
    steps:
      - run:
          shell: true
          command: "echo conversion aborted: bad header 1>&2 & exit 3"
      - assert:
          exit_code: 0

When #

${atago} run stderr_cause.atago.yaml

Then #

  • exit code is 1
  • stdout contains Stderr:, conversion aborted: bad header

Scenario: an exit_code failure falls back to stdout when stderr is silent #

skipped on Windows

Given #

  • Fixture file stdout_cause.atago.yaml is created.

Inputs #

Fixture stdout_cause.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: converter fails on stdout
    steps:
      - run:
          shell: true
          command: "echo wrote 0 of 3 files; exit 3"
      - assert:
          exit_code: 0

When #

${atago} run stdout_cause.atago.yaml

Then #

  • exit code is 1
  • stdout contains Stdout:, wrote 0 of 3 files

Scenario: an exit_code failure falls back to stdout when stderr is silent (windows) #

only on Windows

Given #

  • Fixture file stdout_cause.atago.yaml is created.

Inputs #

Fixture stdout_cause.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: converter fails on stdout
    steps:
      - run:
          shell: true
          command: "echo wrote 0 of 3 files & exit 3"
      - assert:
          exit_code: 0

When #

${atago} run stdout_cause.atago.yaml

Then #

  • exit code is 1
  • stdout contains Stdout:, wrote 0 of 3 files

Scenario: a parse error exits with code two #

Given #

  • Fixture file broken.atago.yaml is created.

Inputs #

Fixture broken.atago.yaml:

version: "1"
suite:
  : not valid yaml

When #

${atago} run broken.atago.yaml

Then #

  • exit code is 2

Scenario: JSON report is valid JSON with a passed status #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: echo
    steps:
      - run:
          shell: true
          command: echo hello
      - assert:
          stdout:
            contains: hello

When #

${atago} run --report json ok.atago.yaml

Then #

  • exit code is 0
  • stdout at $.schema_version equals 1
  • stdout at $.suites[0].status equals passed
  • stdout at $.suites[0].scenarios has length 1

Scenario: a flag after the spec path is still a flag #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: echo
    steps:
      - run:
          shell: true
          command: "exit 0"
      - assert:
          exit_code: 0

When #

${atago} run ok.atago.yaml --report json

Then #

  • exit code is 0
  • stdout at $.suites[0].status equals passed

Scenario: a flag between two spec paths is still a flag #

Given #

  • Fixture file ok.atago.yaml is created.
  • Fixture file second.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: echo
    steps:
      - run:
          shell: true
          command: "exit 0"
      - assert:
          exit_code: 0

Fixture second.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: echo
    steps:
      - run:
          shell: true
          command: "exit 0"
      - assert:
          exit_code: 0

When #

${atago} run ok.atago.yaml --report json second.atago.yaml

Then #

  • exit code is 0
  • stdout at $.suites has length 2

Scenario: a double dash keeps later arguments as paths #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: echo
    steps:
      - run:
          shell: true
          command: "exit 0"
      - assert:
          exit_code: 0

When #

${atago} run -- ok.atago.yaml --report

Then #

  • exit code is 3
  • stderr contains cannot access "--report"

atago self-hosting / sandbox_home (isolated per-OS home) #

Source: test/e2e/atago/sandbox_home.atago.yaml

Scenario: Unix XDG family — write config, read it back, inspect it under the workdir #

skipped on Windows

Given #

  • The command runs with an isolated home under ${workdir}/.atago-home (HOME/XDG or APPDATA redirected).
  • The command runs with an isolated home under ${workdir}/.atago-home (HOME/XDG or APPDATA redirected).

When #

mkdir -p "$XDG_CONFIG_HOME/mytool" && printf editor=vim > "$XDG_CONFIG_HOME/mytool/config"
cat "$XDG_CONFIG_HOME/mytool/config"

Then #

  • after mkdir -p "$XDG_CONFIG_HOME/mytool" && printf editor=vim > "$XDG_CONFIG_HOME/mytool/config":
    • exit code is 0
  • after cat "$XDG_CONFIG_HOME/mytool/config":
    • exit code is 0
    • stdout equals an exact value
    • file .atago-home/.config/mytool/config contains editor=vim

Scenario: Windows APPDATA family — write config, read it back, inspect it under the workdir #

only on Windows

Given #

  • The command runs with an isolated home under ${workdir}/.atago-home (HOME/XDG or APPDATA redirected).
  • The command runs with an isolated home under ${workdir}/.atago-home (HOME/XDG or APPDATA redirected).

When #

mkdir "%APPDATA%\mytool" & echo editor=vim>"%APPDATA%\mytool\config.txt"
type "%APPDATA%\mytool\config.txt"

Then #

  • after mkdir "%APPDATA%\mytool" & echo editor=vim>"%APPDATA%\mytool\config.txt":
    • exit code is 0
  • after type "%APPDATA%\mytool\config.txt":
    • exit code is 0
    • stdout contains editor=vim
    • file .atago-home/AppData/Roaming/mytool/config.txt contains editor=vim

Scenario: cwd anchors the run, but sandbox_home stays at the workdir ROOT (Unix) #

skipped on Windows

Given #

  • The command runs with an isolated home under ${workdir}/.atago-home (HOME/XDG or APPDATA redirected).

When #

mkdir -p sub
mkdir -p "$XDG_CONFIG_HOME/mytool" && printf editor=vim > "$XDG_CONFIG_HOME/mytool/config"

Then #

  • after mkdir -p "$XDG_CONFIG_HOME/mytool" && printf editor=vim > "$XDG_CONFIG_HOME/mytool/config":
    • exit code is 0
    • file .atago-home/.config/mytool/config contains editor=vim
    • file sub/.atago-home/.config/mytool/config does not exist

verbatim scalars #

Source: test/e2e/atago/scalar_verbatim.atago.yaml

Scenario: unquoted literals keep their digits in matchers, env, and fixtures #

Given #

  • Fixture file child.atago.yaml is created.

Inputs #

Fixture child.atago.yaml:

version: "1"
suite: {name: child}
scenarios:
  - name: a trailing zero survives into an exact match
    steps:
      - run: {command: echo 1.20}
      - assert: {stdout: {equals: "1.20"}}
  - name: leading zeros reach the program under test
    steps:
      - run: {shell: true, command: 'echo "[$V]"', env: {V: 007}}
      - assert: {stdout: {equals: "[007]"}}
  - name: a fixture keeps the digits it was written with
    steps:
      - fixture: {file: version.txt, content: 1.10}
      - assert: {file: {path: version.txt, equals: "1.10"}}
  - name: a boolean-looking command still names the program
    steps:
      - run: {command: true}
      - assert: {exit_code: 0}

When #

${atago} run --parallel 1 child.atago.yaml

Then #

  • exit code is 0
  • stdout contains 4 passed

Scenario: output that lacks the literal fails instead of matching a shortened one #

Given #

  • Fixture file child.atago.yaml is created.

Inputs #

Fixture child.atago.yaml:

version: "1"
suite: {name: child}
scenarios:
  - name: a
    steps:
      - run: {command: echo 1.2}
      - assert: {stdout: {contains: 1.20}}

When #

${atago} run child.atago.yaml

Then #

  • exit code is 1
  • stdout contains 1.20

Scenario: a json matcher keeps YAML’s typing, so true is not the text true #

When #

echo '{"ok": true, "name": "true"}'

Then #

  • stdout at $.ok equals true; at $.name equals true

atago self-hosting / security #

Source: test/e2e/atago/security.atago.yaml

Scenario: declared secrets are masked in failure output #

Given #

  • Fixture file sec.atago.yaml is created.
  • Environment variables are set: DEMO_TOKEN.

Inputs #

Fixture sec.atago.yaml:

version: "1"
suite:
  name: inner
secrets:
  - DEMO_TOKEN
scenarios:
  - name: leaky
    steps:
      - run:
          shell: true
          command: echo "token=$DEMO_TOKEN"
      - assert:
          stdout:
            contains: NOPE

When #

${atago} run sec.atago.yaml

Then #

  • exit code is 1
  • stdout contains token=***

Scenario: a file assertion path may not escape the scenario workdir #

Given #

  • Fixture file escape.atago.yaml is created.

Inputs #

Fixture escape.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: traversal
    steps:
      - run:
          shell: true
          command: echo hi
      - assert:
          file:
            path: ../../../etc/hosts
            exists: true

When #

${atago} run escape.atago.yaml

Then #

  • exit code is 1
  • stdout contains escapes the scenario workdir

Scenario: a file assertion may not read through a symlinked directory #

skipped on Windows

Given #

  • Fixture file link_read.atago.yaml is created.

Inputs #

Fixture link_read.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: read through the link
    steps:
      - run:
          command: ln -s .. escape
      - assert:
          file:
            path: escape/anything.txt
            exists: false

When #

${atago} run link_read.atago.yaml

Then #

  • exit code is 1
  • stdout contains escapes the scenario workdir

Scenario: a redirect may not write through a symlinked directory #

skipped on Windows

Given #

  • Fixture file link_write.atago.yaml is created.

Inputs #

Fixture link_write.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: write through the link
    steps:
      - run:
          command: ln -s .. escape
      - run:
          shell: true
          command: echo planted
          stdout_to: escape/planted.txt

When #

${atago} run link_write.atago.yaml

Then #

  • exit code is 4
  • stdout contains escapes the scenario workdir

Scenario: a symlinked directory inside the workdir still resolves #

skipped on Windows

Given #

  • Fixture file link_ok.atago.yaml is created.

Inputs #

Fixture link_ok.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: read and write through an in-workdir link
    steps:
      - run:
          command: mkdir real
      - run:
          command: ln -s real alias
      - run:
          shell: true
          command: echo through the alias
          stdout_to: alias/out.txt
      - assert:
          file:
            path: real/out.txt
            contains: through the alias

When #

${atago} run link_ok.atago.yaml

Then #

  • exit code is 0

Scenario: a snapshot path may not escape the spec directory #

Given #

  • Fixture file snap_escape.atago.yaml is created.

Inputs #

Fixture snap_escape.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: traversal
    steps:
      - run:
          shell: true
          command: echo hi
      - assert:
          stdout:
            snapshot: ../../../tmp/leak.snap

When #

${atago} run snap_escape.atago.yaml

Then #

  • exit code is 1
  • stdout contains escapes the spec directory

atago self-hosting / selection #

Source: test/e2e/atago/select.atago.yaml

Scenario: –filter runs only matching scenarios #

Given #

  • Fixture file many.atago.yaml is created.

Inputs #

Fixture many.atago.yaml:

version: "1"
suite:
  name: many
scenarios:
  - name: keep this one
    steps: [{run: {shell: true, command: echo a}}, {assert: {exit_code: 0}}]
  - name: drop that one
    steps: [{run: {shell: true, command: echo b}}, {assert: {exit_code: 0}}]

When #

${atago} run --filter keep many.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: –filter selects multiple scenarios with OR (comma and repeated) #

Given #

  • Fixture file three.atago.yaml is created.

Inputs #

Fixture three.atago.yaml:

version: "1"
suite:
  name: three
scenarios:
  - name: alpha one
    steps: [{run: {shell: true, command: echo a}}, {assert: {exit_code: 0}}]
  - name: beta two
    steps: [{run: {shell: true, command: echo b}}, {assert: {exit_code: 0}}]
  - name: gamma three
    steps: [{run: {shell: true, command: echo c}}, {assert: {exit_code: 0}}]

When #

${atago} run --filter alpha,beta three.atago.yaml
${atago} run --filter alpha --filter gamma three.atago.yaml

Then #

  • after ${atago} run --filter alpha,beta three.atago.yaml:
    • exit code is 0
    • stdout contains 2 passed
  • after ${atago} run --filter alpha --filter gamma three.atago.yaml:
    • exit code is 0
    • stdout contains 2 passed

Scenario: –skip-tag drops tagged scenarios #

Given #

  • Fixture file tagged.atago.yaml is created.

Inputs #

Fixture tagged.atago.yaml:

version: "1"
suite:
  name: tagged
scenarios:
  - name: quick
    tags: [fast]
    steps: [{run: {shell: true, command: echo a}}, {assert: {exit_code: 0}}]
  - name: heavy
    tags: [slow]
    steps: [{run: {shell: true, command: echo b}}, {assert: {exit_code: 0}}]

When #

${atago} run --skip-tag slow tagged.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: a repeated tag on one scenario is a load-time error #

Given #

  • Fixture file duptag.atago.yaml is created.
  • Fixture file shared.atago.yaml is created.

Inputs #

Fixture duptag.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: tagged twice
    tags: [smoke, smoke]
    steps:
      - run: {shell: true, command: "true"}

Fixture shared.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: first
    tags: [smoke]
    steps:
      - run: {shell: true, command: "true"}
  - name: second
    tags: [smoke]
    steps:
      - run: {shell: true, command: "true"}

When #

${atago} run duptag.atago.yaml
${atago} doc shared.atago.yaml

Then #

  • after ${atago} run duptag.atago.yaml:
    • exit code is 2
    • stderr contains duplicate tag "smoke"
  • after ${atago} doc shared.atago.yaml:
    • exit code is 0
    • stdout contains `smoke` (2)

Scenario: an empty tag is a load-time error #

Given #

  • Fixture file emptytag.atago.yaml is created.

Inputs #

Fixture emptytag.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: carries an empty tag
    tags: ["", smoke]
    steps:
      - run: {shell: true, command: "true"}

When #

${atago} run emptytag.atago.yaml

Then #

  • exit code is 2
  • stderr contains tag must not be empty

Scenario: a ready.store that shadows a built-in is a load-time error #

Given #

  • Fixture file readyshadow.atago.yaml is created.

Inputs #

Fixture readyshadow.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: shadows the workdir
    services:
      - name: writer
        command: ./writer
        ready: {file: marker.txt, store: workdir}
    steps:
      - run: {shell: true, command: "true"}

When #

${atago} run readyshadow.atago.yaml

Then #

  • exit code is 2
  • stderr contains shadows a built-in variable

Scenario: a scenario service that shadows a suite service is a load-time error #

Given #

  • Fixture file svcshadow.atago.yaml is created.

Inputs #

Fixture svcshadow.atago.yaml:

version: "1"
suite:
  name: inner
  setup:
    - service: {name: peer, command: ./peer}
scenarios:
  - name: declares its own peer
    services:
      - {name: peer, command: ./peer}
    steps:
      - run: {shell: true, command: "true"}

When #

${atago} run svcshadow.atago.yaml

Then #

  • exit code is 2
  • stderr contains duplicate service name "peer"

atago self-hosting / background services #

Source: test/e2e/atago/services.atago.yaml

Scenario: file readiness captures a dynamic value into a variable #

Given #

  • Background service publisher is started: printf "127.0.0.1:5555" > ready.txt; sleep 30.

When #

echo ${addr}

Then #

  • stdout equals an exact value

Scenario: log readiness waits for a line on the service output #

Given #

  • Background service logger is started: echo "ready: listening"; sleep 30.

When #

echo started

Then #

  • stdout contains started

Scenario: delay readiness waits a fixed duration #

Given #

  • Background service slow is started: sleep 30.

When #

echo ok

Then #

  • stdout contains ok

Scenario: multiple services start and capture independently #

Given #

  • Background service first is started: printf alpha > a.txt; sleep 30.
  • Background service second is started: printf beta > b.txt; sleep 30.

When #

echo ${a}-${b}

Then #

  • stdout equals an exact value

Scenario: a readiness failure preserves the service log as an artifact #

Given #

  • Fixture file notready.atago.yaml is created.

Inputs #

Fixture notready.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: service never ready
    services:
      - name: chatty
        shell: true
        command: 'echo booting-up; sleep 30'
        ready:
          file: never.txt
          timeout: 200ms
    steps:
      - run:
          command: echo unreached

When #

${atago} run --artifacts-dir arts notready.atago.yaml
cat arts/*/*/service-chatty.log

Then #

  • after ${atago} run --artifacts-dir arts notready.atago.yaml:
    • exit code is 4
  • after cat arts/*/*/service-chatty.log:
    • exit code is 0
    • stdout contains booting-up

Scenario: a step failure after the service is ready preserves the service log #

Given #

  • Fixture file readythenfail.atago.yaml is created.

Inputs #

Fixture readythenfail.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: ready then fail
    services:
      - name: peer
        shell: true
        command: 'echo peer-serving > ready.txt; echo peer-log-line; sleep 30'
        ready:
          file: ready.txt
          timeout: 2s
    steps:
      - run:
          command: echo hello
      - assert:
          stdout:
            contains: goodbye

When #

${atago} run --report json --artifacts-dir arts readythenfail.atago.yaml
cat arts/*/*/service-peer.log

Then #

  • after ${atago} run --report json --artifacts-dir arts readythenfail.atago.yaml:
    • exit code is 1
    • stdout contains service_logs, service-peer.log
  • after cat arts/*/*/service-peer.log:
    • exit code is 0
    • stdout contains peer-log-line

Scenario: a green run with a healthy service writes no service log #

Given #

  • Fixture file healthy.atago.yaml is created.

Inputs #

Fixture healthy.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: healthy service passes
    services:
      - name: ok
        shell: true
        command: 'echo up > ready.txt; echo serving; sleep 30'
        ready:
          file: ready.txt
          timeout: 2s
    steps:
      - run:
          command: echo hi
      - assert:
          stdout:
            contains: hi

When #

${atago} run --artifacts-dir arts healthy.atago.yaml
ls arts 2>/dev/null | wc -l | tr -d " "

Then #

  • after ${atago} run --artifacts-dir arts healthy.atago.yaml:
    • exit code is 0
  • after ls arts 2>/dev/null | wc -l | tr -d " ":
    • stdout equals an exact value

Scenario: a service that dies before readiness names its exit status #

skipped on Windows

Given #

  • Fixture file crasher.atago.yaml is created.

Inputs #

Fixture crasher.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: service exits with a status
    services:
      - name: dies
        shell: true
        command: 'echo starting; exit 3'
        ready:
          log: never-appears
          timeout: 2s
    steps:
      - run:
          command: echo unreached

When #

${atago} run crasher.atago.yaml

Then #

  • exit code is 4
  • stdout contains service exited before it became ready (exit status 3), starting

atago self-hosting / the harness shell cannot be supplied by the program under test #

Source: test/e2e/atago/shell_path.atago.yaml

Scenario: a shell planted on the inner atago’s PATH does not hijack shell:true #

Given #

  • Fixture file sh is created.
  • Fixture file cmd.bat is created.
  • Fixture file inner.atago.yaml is created.
  • Environment variables are set: PATH.

Inputs #

Fixture sh:

#!/bin/sh
echo HIJACKED
exit 0

Fixture cmd.bat:

@echo HIJACKED

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: the shell is the system one
    steps:
      - run:
          shell: true
          command: "echo real-shell"
      - assert:
          exit_code: 0
          stdout:
            equals: real-shell

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 0
  • stdout does not contain HIJACKED

Scenario: ATAGO_SHELL selects the shell the inner atago runs shell:true with #

Given #

  • Fixture file inner.atago.yaml is created.
  • Environment variables are set: ATAGO_SHELL.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: a shell step
    steps:
      - run:
          shell: true
          command: "echo hi"

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 4
  • stdout contains no-such-shell

atago self-hosting / signal step (graceful shutdown) #

Source: test/e2e/atago/signal.atago.yaml

Scenario: SIGTERM reaches the trap handler and wait observes the exit #

skipped on Windows

Given #

  • Background service server is started: trap 'echo graceful shutdown complete > server.log; exit 0' TERM; echo booted; while true; do sleep 0.1; done.

When #

# send SIGTERM to service server and wait up to 5s for exit

Then #

  • file server.log contains graceful shutdown complete

Scenario: SIGHUP triggers a reload without stopping the service #

skipped on Windows

Given #

  • Background service reloader is started: trap 'echo reloaded >> reload.log' HUP; echo booted; while true; do sleep 0.1; done.

When #

# send SIGHUP to service reloader
for i in $(seq 1 100); do grep -q reloaded reload.log 2>/dev/null && break; sleep 0.1; done; cat reload.log

Then #

  • after for i in $(seq 1 100); do grep -q reloaded reload.log 2>/dev/null && break; sleep 0.1; done; cat reload.log:
    • exit code is 0
    • stdout contains reloaded

Scenario: a wait timeout on a TERM-ignoring service fails with the documented message #

skipped on Windows

Given #

  • Fixture file stubborn.atago.yaml is created.

Inputs #

Fixture stubborn.atago.yaml:

version: "1"
suite:
  name: stubborn
scenarios:
  - name: never exits on TERM
    services:
      - name: stubborn
        shell: true
        command: "trap '' TERM; echo booted; while true; do sleep 0.2; done"
        ready: {log: booted, timeout: 10s}
    steps:
      - signal:
          service: stubborn
          signal: TERM
          wait:
            timeout: 300ms

When #

${atago} run stubborn.atago.yaml

Then #

  • exit code is not 0
  • stdout contains did not exit within 300ms after SIGTERM

Scenario: an unknown target service is a load-time error listing declared names #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: wrong name
    services:
      - name: web
        command: ./web
    steps:
      - signal:
          service: cache
          signal: TERM

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains not a declared service (declared: web)

atago self-hosting / skip-only command predicate #

Source: test/e2e/atago/skip_command.atago.yaml

Scenario: skip command that succeeds skips the scenario #

Given #

  • Fixture file skip.atago.yaml is created.

Inputs #

Fixture skip.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: gated by a succeeding probe
    skip:
      command: "exit 0"
    steps:
      - run:
          shell: true
          command: echo should-not-run
      - assert:
          stdout:
            contains: never-checked

When #

${atago} run skip.atago.yaml

Then #

  • exit code is 0
  • stdout contains skipped

Scenario: only command that fails skips the scenario #

Given #

  • Fixture file only.atago.yaml is created.

Inputs #

Fixture only.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: gated by a failing probe
    only:
      command: "exit 1"
    steps:
      - run:
          shell: true
          command: echo should-not-run
      - assert:
          stdout:
            contains: never-checked

When #

${atago} run only.atago.yaml

Then #

  • exit code is 0
  • stdout contains skipped

Scenario: only command that succeeds runs the scenario #

Given #

  • Fixture file run.atago.yaml is created.

Inputs #

Fixture run.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: gated by a succeeding probe
    only:
      command: "exit 0"
    steps:
      - run:
          shell: true
          command: echo ran
      - assert:
          stdout:
            contains: ran

When #

${atago} run run.atago.yaml

Then #

  • exit code is 0
  • stdout contains passed

atago self-hosting / snapshot #

Source: test/e2e/atago/snapshot.atago.yaml

Scenario: a snapshot assertion passes against a committed snapshot #

Given #

  • Fixture file snap.atago.yaml is created.
  • Fixture file out.snap is created.

Inputs #

Fixture snap.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: stable output
    steps:
      - run:
          command: echo stable
      - assert:
          stdout:
            snapshot: out.snap

Fixture out.snap:

stable

When #

${atago} run snap.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: snapshot update creates the snapshot file #

Given #

  • Fixture file gen.atago.yaml is created.

Inputs #

Fixture gen.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: stable output
    steps:
      - run:
          command: echo stable
      - assert:
          stdout:
            snapshot: created.snap

When #

${atago} snapshot update gen.atago.yaml

Then #

  • exit code is 0
  • file created.snap contains stable

Scenario: a snapshot mismatch writes the normalized actual as an artifact #

Given #

  • Fixture file drift.atago.yaml is created.
  • Fixture file committed.snap is created.

Inputs #

Fixture drift.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: drifted output
    steps:
      - run:
          command: echo changed
      - assert:
          stdout:
            snapshot: committed.snap

Fixture committed.snap:

original

When #

${atago} run --artifacts-dir arts drift.atago.yaml
cat arts/*/*/step-*-snapshot.actual.txt

Then #

  • after ${atago} run --artifacts-dir arts drift.atago.yaml:
    • exit code is 1
    • stdout contains Artifacts:
  • after cat arts/*/*/step-*-snapshot.actual.txt:
    • exit code is 0
    • stdout contains changed

atago self-hosting / snapshot normalization and round-trip #

Source: test/e2e/atago/snapshot_normalization.atago.yaml

Scenario: record then run round-trips green #

Given #

  • Fixture file rt.atago.yaml is created.

Inputs #

Fixture rt.atago.yaml:

version: "1"
suite: {name: rt}
scenarios:
  - name: stable line
    steps:
      - run: {shell: true, command: "printf 'stable output line\\n'"}
      - assert: {stdout: {snapshot: g.txt}}

When #

${atago} snapshot update rt.atago.yaml
${atago} run rt.atago.yaml

Then #

  • after ${atago} snapshot update rt.atago.yaml:
    • exit code is 0
  • after ${atago} run rt.atago.yaml:
    • exit code is 0

Scenario: a UUID is masked in the golden #

Given #

  • Fixture file uuid.atago.yaml is created.

Inputs #

Fixture uuid.atago.yaml:

version: "1"
suite: {name: uuid}
scenarios:
  - name: emits a uuid
    steps:
      - run: {shell: true, command: "printf 'id=550e8400-e29b-41d4-a716-446655440000\\n'"}
      - assert: {stdout: {snapshot: g.txt}}

When #

${atago} snapshot update uuid.atago.yaml

Then #

  • file g.txt contains id=<uuid>
  • file g.txt does not contain 550e8400

Scenario: an ISO timestamp is masked in the golden #

Given #

  • Fixture file ts.atago.yaml is created.

Inputs #

Fixture ts.atago.yaml:

version: "1"
suite: {name: ts}
scenarios:
  - name: emits a timestamp
    steps:
      - run: {shell: true, command: "printf 'at 2024-01-15T10:30:00Z done\\n'"}
      - assert: {stdout: {snapshot: g.txt}}

When #

${atago} snapshot update ts.atago.yaml

Then #

  • file g.txt contains at <timestamp> done

Scenario: a loopback host and port are masked in the golden #

Given #

  • Fixture file port.atago.yaml is created.

Inputs #

Fixture port.atago.yaml:

version: "1"
suite: {name: port}
scenarios:
  - name: emits a listen address
    steps:
      - run: {shell: true, command: "printf 'listening on 127.0.0.1:54321\\n'"}
      - assert: {stdout: {snapshot: g.txt}}

When #

${atago} snapshot update port.atago.yaml

Then #

  • file g.txt contains 127.0.0.1:<port>
  • file g.txt does not contain 54321

Scenario: the home directory is masked to a tilde in the golden #

skipped on Windows

Given #

  • Fixture file home.atago.yaml is created.

Inputs #

Fixture home.atago.yaml:

version: "1"
suite: {name: home}
scenarios:
  - name: emits the home path
    steps:
      - run: {shell: true, command: "printf 'home=%s/x\\n' \"$HOME\""}
      - assert: {stdout: {snapshot: g.txt}}

When #

${atago} snapshot update home.atago.yaml

Then #

  • file g.txt contains home=~/x

Scenario: an escape between a CR and an LF leaves no CR in the golden #

Given #

  • Fixture file cr.atago.yaml is created.

Inputs #

Fixture cr.atago.yaml:

version: "1"
suite: {name: cr}
scenarios:
  - name: rewrites a progress line
    steps:
      - run: {shell: true, command: "printf 'working\\r\\033[K\\ndone\\n'"}
      - assert: {stdout: {snapshot: g.txt}}

When #

${atago} snapshot update cr.atago.yaml
${atago} run cr.atago.yaml

Then #

  • after ${atago} snapshot update cr.atago.yaml:
    • exit code is 0
    • file g.txt does not contain "\r"
  • after ${atago} run cr.atago.yaml:
    • exit code is 0

Scenario: a golden verifies against a different volatile value #

Given #

  • Fixture file rec.atago.yaml is created.
  • Fixture file ver.atago.yaml is created.

Inputs #

Fixture rec.atago.yaml:

version: "1"
suite: {name: rec}
scenarios:
  - name: uuid A
    steps:
      - run: {shell: true, command: "printf 'token=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\\n'"}
      - assert: {stdout: {snapshot: shared.txt}}

Fixture ver.atago.yaml:

version: "1"
suite: {name: rec}
scenarios:
  - name: uuid A
    steps:
      - run: {shell: true, command: "printf 'token=11111111-2222-3333-4444-555555555555\\n'"}
      - assert: {stdout: {snapshot: shared.txt}}

When #

${atago} snapshot update rec.atago.yaml
${atago} run ver.atago.yaml

Then #

  • after ${atago} snapshot update rec.atago.yaml:
    • exit code is 0
  • after ${atago} run ver.atago.yaml:
    • exit code is 0

Scenario: updating a snapshot is deterministic #

Given #

  • Fixture file d1.atago.yaml is created.
  • Fixture file d2.atago.yaml is created.

Inputs #

Fixture d1.atago.yaml:

version: "1"
suite: {name: d}
scenarios:
  - name: same command
    steps:
      - run: {shell: true, command: "printf 'line one\\nline two\\n'"}
      - assert: {stdout: {snapshot: a.txt}}

Fixture d2.atago.yaml:

version: "1"
suite: {name: d}
scenarios:
  - name: same command
    steps:
      - run: {shell: true, command: "printf 'line one\\nline two\\n'"}
      - assert: {stdout: {snapshot: b.txt}}

When #

${atago} snapshot update d1.atago.yaml
${atago} snapshot update d2.atago.yaml

Then #

  • after ${atago} snapshot update d2.atago.yaml:
    • file a.txt is byte-identical to b.txt

Scenario: a real content change still fails the snapshot #

Given #

  • Fixture file change.atago.yaml is created.
  • Fixture file g.txt is created.

Inputs #

Fixture change.atago.yaml:

version: "1"
suite: {name: change}
scenarios:
  - name: drifts from the golden
    steps:
      - run: {shell: true, command: "printf 'the committed line\\n'"}
      - assert: {stdout: {snapshot: g.txt}}

Fixture g.txt:

a different committed line

When #

${atago} run change.atago.yaml

Then #

  • exit code is 1

Scenario: a missing golden names the update flag #

Given #

  • Fixture file miss.atago.yaml is created.

Inputs #

Fixture miss.atago.yaml:

version: "1"
suite: {name: miss}
scenarios:
  - name: no golden yet
    steps:
      - run: {shell: true, command: "printf 'hi\\n'"}
      - assert: {stdout: {snapshot: never.txt}}

When #

${atago} run miss.atago.yaml

Then #

  • exit code is 1
  • stdout contains --update-snapshots

atago self-hosting / snapshots #

Source: test/e2e/atago/snapshots.atago.yaml

Scenario: two scenarios writing different content to one snapshot path fail the update #

Given #

  • Fixture file clash.atago.yaml is created.
  • Fixture file shared_golden.atago.yaml is created.

Inputs #

Fixture clash.atago.yaml:

version: "1"
suite:
  name: clash
scenarios:
  - name: alpha
    steps:
      - run: {shell: true, command: "echo from-alpha"}
      - assert: {stdout: {snapshot: shared.snap}}
  - name: beta
    steps:
      - run: {shell: true, command: "echo from-beta"}
      - assert: {stdout: {snapshot: shared.snap}}

Fixture shared_golden.atago.yaml:

version: "1"
suite:
  name: shared
scenarios:
  - name: short flag
    steps:
      - run: {shell: true, command: "echo usage"}
      - assert: {stdout: {snapshot: help.snap}}
  - name: long flag
    steps:
      - run: {shell: true, command: "echo usage"}
      - assert: {stdout: {snapshot: help.snap}}

When #

${atago} run --parallel 1 --update-snapshots clash.atago.yaml
${atago} run --parallel 1 --update-snapshots shared_golden.atago.yaml
${atago} run --parallel 1 shared_golden.atago.yaml

Then #

  • after ${atago} run --parallel 1 --update-snapshots clash.atago.yaml:
    • exit code is 1
    • stdout contains already written in this run with different content
  • after ${atago} run --parallel 1 --update-snapshots shared_golden.atago.yaml:
    • exit code is 0
  • after ${atago} run --parallel 1 shared_golden.atago.yaml:
    • exit code is 0
    • stdout contains 2 passed
    • stdout does not contain 2 snapshots updated

Scenario: the rewrite count covers teardown and the suite lifecycle #

Given #

  • Fixture file lifecycle.atago.yaml is created.

Inputs #

Fixture lifecycle.atago.yaml:

version: "1"
suite:
  name: lifecycle
  setup:
    - run: {shell: true, command: "echo setup banner"}
    - assert: {stdout: {snapshot: setup.snap}}
  teardown:
    - run: {shell: true, command: "echo suite teardown"}
    - assert: {stdout: {snapshot: suite_teardown.snap}}
scenarios:
  - name: writes goldens in every phase
    steps:
      - run: {shell: true, command: "echo main output"}
      - assert: {stdout: {snapshot: main.snap}}
    teardown:
      - run: {shell: true, command: "echo scenario teardown"}
      - assert: {stdout: {snapshot: scenario_teardown.snap}}

When #

${atago} run --parallel 1 --update-snapshots lifecycle.atago.yaml
${atago} run --parallel 1 lifecycle.atago.yaml

Then #

  • after ${atago} run --parallel 1 --update-snapshots lifecycle.atago.yaml:
    • exit code is 0
    • stdout contains 4 snapshots updated
    • file setup.snap contains setup banner
    • file suite_teardown.snap contains suite teardown
    • file scenario_teardown.snap contains scenario teardown
  • after ${atago} run --parallel 1 lifecycle.atago.yaml:
    • exit code is 0

Scenario: a red run still reports the goldens it rewrote #

skipped on Windows

Given #

  • Fixture file unstable.atago.yaml is created.

Inputs #

Fixture unstable.atago.yaml:

version: "1"
suite:
  name: unstable
scenarios:
  - name: output differs between iterations
    steps:
      - run: {shell: true, command: "date +%s%N"}
      - assert: {stdout: {snapshot: now.snap}}

When #

${atago} run --parallel 1 --repeat 2 --update-snapshots unstable.atago.yaml

Then #

  • exit code is 1
  • stdout contains 1 snapshot updated
  • file now.snap exists

Generated artifacts #

  • now.snap

Scenario: a clash with the scenario’s own earlier attempt names the attempt #

skipped on Windows

Given #

  • Fixture file retry.atago.yaml is created.

Inputs #

Fixture retry.atago.yaml:

version: "1"
suite:
  name: retry
scenarios:
  - name: snapshots a value that changes every attempt
    steps:
      - run: {shell: true, command: "date +%s%N"}
      - assert: {stdout: {snapshot: attempt.snap}}
      - run: {shell: true, command: "exit 1"}
      - assert: {exit_code: 0}

When #

${atago} run --parallel 1 --retry-failed 1 --update-snapshots retry.atago.yaml

Then #

  • exit code is 1
  • stdout contains changed between attempts of this scenario, does not contain two scenarios cannot share

atago self-hosting / ssh runner #

Source: test/e2e/atago/ssh.atago.yaml

Scenario: an ssh runner without host/user fails validation (exit 2) #

Given #

  • Fixture file badssh.atago.yaml is created.

Inputs #

Fixture badssh.atago.yaml:

version: "1"
suite:
  name: ssh
runners:
  box:
    type: ssh
scenarios:
  - name: remote run
    steps:
      - run:
          runner: box
          command: uptime

When #

${atago} run badssh.atago.yaml

Then #

  • exit code is 2
  • stderr contains requires a host

Scenario: a run step naming an undeclared runner fails validation (exit 2) #

Given #

  • Fixture file norunner.atago.yaml is created.

Inputs #

Fixture norunner.atago.yaml:

version: "1"
suite:
  name: ssh
scenarios:
  - name: remote run
    steps:
      - run:
          runner: missing
          command: uptime

When #

${atago} run norunner.atago.yaml

Then #

  • exit code is 2
  • stderr contains is not declared

Scenario: a local-only run field on an ssh runner fails validation (exit 2) #

Given #

  • Fixture file sshfield.atago.yaml is created.

Inputs #

Fixture sshfield.atago.yaml:

version: "1"
suite:
  name: ssh
runners:
  box:
    type: ssh
    host: deploy.example.com
    user: deploy
scenarios:
  - name: remote run
    steps:
      - run:
          runner: box
          command: uptime
          cwd: /var/tmp

When #

${atago} run sshfield.atago.yaml

Then #

  • exit code is 2
  • stderr contains run.cwd has no effect on an ssh runner, steps[0].run

atago self-hosting / stdin sources (file + base64) #

Source: test/e2e/atago/stdin_sources.atago.yaml

Scenario: base64 stdin delivers the exact byte count #

skipped on Windows

Inputs #

stdin for wc:

(binary, 8 base64 chars)

When #

wc -c

Then #

  • exit code is 0
  • stdout matches /^\s*4\s*$/

Scenario: stdin file is expanded and read from the workdir #

skipped on Windows

Given #

  • Fixture file payload.txt is created.

Inputs #

Fixture payload.txt:

from-a-file

stdin for cat:

(read from file ${workdir}/payload.txt)

When #

cat

Then #

  • exit code is 0
  • stdout equals an exact value

Scenario: a stdin file outside the workdir is rejected at runtime #

skipped on Windows

Given #

  • Fixture file escape.atago.yaml is created.

Inputs #

Fixture escape.atago.yaml:

version: "1"
suite:
  name: escape
scenarios:
  - name: tries to read outside the workdir
    steps:
      - run:
          command: cat
          stdin:
            file: ../outside.txt

When #

${atago} run escape.atago.yaml

Then #

  • exit code is not 0
  • stdout contains run.stdin.file

Scenario: stdin with both file and base64 is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
scenarios:
  - name: both sources
    steps:
      - run:
          command: cat
          stdin:
            file: in.txt
            base64: AAEC

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains exactly one of file/base64

Scenario: invalid base64 stdin is a load-time error #

Given #

  • Fixture file badb64.atago.yaml is created.

Inputs #

Fixture badb64.atago.yaml:

version: "1"
suite:
  name: badb64
scenarios:
  - name: bad payload
    steps:
      - run:
          command: cat
          stdin:
            base64: "not!!base64"

When #

${atago} run badb64.atago.yaml

Then #

  • exit code is 2
  • stderr contains not valid base64

atago self-hosting / store #

Source: test/e2e/atago/store.atago.yaml

Scenario: a stored JSON value is reusable in later commands #

Given #

  • Fixture file store.atago.yaml is created.

Inputs #

Fixture store.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: capture id and greet
    steps:
      - run:
          shell: true
          command: printf '{"id":99,"name":"Bob"}'
      - store:
          name: user_id
          from:
            stdout:
              json:
                path: "$.id"
      - run:
          shell: true
          command: echo "user is ${user_id}"
      - assert:
          stdout:
… (truncated, 1 more line)

When #

${atago} run store.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASSED

Scenario: storing from a missing JSON path is an execution error #

Given #

  • Fixture file bad-store.atago.yaml is created.

Inputs #

Fixture bad-store.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: capture nonexistent
    steps:
      - run:
          shell: true
          command: printf '{}'
      - store:
          name: missing
          from:
            stdout:
              json:
                path: "$.nope"
      - assert:
          exit_code: 0

When #

${atago} run bad-store.atago.yaml

Then #

  • exit code is 4
  • stdout contains ERROR

atago self-hosting / store capture boundary values #

Source: test/e2e/atago/store_edges.atago.yaml

Scenario: a regex with a capture group stores the group #

When #

echo release v1.2.3-rc1
# capture ${ver} from stdout
echo shipping ${ver}

Then #

  • after echo shipping ${ver}:
    • stdout equals an exact value

Scenario: a regex without a group stores the whole match #

When #

echo digest sha=abc123def456
# capture ${sha} from stdout
echo checksum ${sha}

Then #

  • after echo checksum ${sha}:
    • stdout equals an exact value

Scenario: a JSON path captures a scalar from stdout #

When #

printf '{"port":8080}'
# capture ${port} from stdout
echo bound ${port}

Then #

  • after echo bound ${port}:
    • stdout equals an exact value

Scenario: a JSON path captures a value from a generated file #

Given #

  • Fixture file meta.json is created.

Inputs #

Fixture meta.json:

{"build":{"id":"b-42"}}

When #

# capture ${build} from file meta.json
echo build ${build}

Then #

  • stdout equals an exact value

Scenario: a regex that matches nothing is an execution error #

Given #

  • Fixture file nomatch.atago.yaml is created.

Inputs #

Fixture nomatch.atago.yaml:

version: "1"
suite: {name: nomatch}
scenarios:
  - name: capture fails
    steps:
      - run: {shell: true, command: "echo hello"}
      - store: {name: x, from: {stdout: {matches: "ABSENT([0-9]+)"}}}

When #

${atago} run nomatch.atago.yaml

Then #

  • exit code is 4
  • stdout contains did not match

Scenario: a stored value does not leak into the next scenario #

Given #

  • Fixture file scope.atago.yaml is created.

Inputs #

Fixture scope.atago.yaml:

version: "1"
suite: {name: scope}
scenarios:
  - name: captures a value
    steps:
      - run: {shell: true, command: "echo secret-value"}
      - store: {name: captured, from: {stdout: {trim: true}}}
      - assert: {stdout: {contains: secret-value}}
  - name: cannot reference the earlier value
    steps:
      - run: {command: "echo ${captured}"}

When #

${atago} run scope.atago.yaml

Then #

  • exit code is 4
  • stdout contains references ${captured}, no variable with that name is defined

atago self-hosting / store whole-content trim and text selectors (#158) #

Source: test/e2e/atago/store_whole.atago.yaml

Scenario: trim captures an opaque token and round-trips it as an argument #

When #

echo opaque-token-abc123
# capture ${token} from stdout
${atago} run ${token}

Then #

  • after ${atago} run ${token}:
    • exit code is 3
    • stderr contains opaque-token-abc123

Scenario: text captures a whole multi-line file verbatim #

Given #

  • Fixture file blob.txt is created.
  • Fixture file copy.txt is created.

Inputs #

Fixture blob.txt:

first line
second line

Fixture copy.txt:

${blob}

When #

# capture ${blob} from file blob.txt

Then #

  • file copy.txt contains first line, second line

atago self-hosting / stream matcher boundary values #

Source: test/e2e/atago/stream_edges.atago.yaml

Scenario: equals a multibyte and emoji line #

When #

printf 'テスト🎌\n'

Then #

  • stdout equals an exact value

Scenario: contains a multibyte substring inside a longer line #

When #

printf 'café ☕ の résumé\n'

Then #

  • stdout contains ☕ の

Scenario: a regex matches across multibyte runes #

When #

printf 'αβγδ\n'

Then #

  • stdout matches /β.δ/

Scenario: line selection returns a multibyte line intact #

When #

printf 'ひらがな\nカタカナ\n漢字\n'

Then #

  • stdout line 2 equals an exact value

Scenario: not_contains a multibyte needle that is absent #

When #

printf '日本語\n'

Then #

  • stdout does not contain 中文

Scenario: empty is true for a command that prints nothing #

When #

true

Then #

  • stdout is empty

Scenario: empty is true for whitespace-only output #

When #

printf '   \n\t\n'

Then #

  • stdout is empty

Scenario: an empty-false failure reveals whitespace-only bytes #

Given #

  • Fixture file inner_ws.atago.yaml is created.

Inputs #

Fixture inner_ws.atago.yaml:

version: "1"
suite:
  name: inner ws
scenarios:
  - name: whitespace is empty
    steps:
      - run: {shell: true, command: 'printf "  \n\n"'}
      - assert:
          stdout:
            empty: false

When #

${atago} run inner_ws.atago.yaml

Then #

  • exit code is 1
  • stdout contains " \n\n"

Scenario: equals tolerates output with no trailing newline #

When #

printf 'noeol'

Then #

  • stdout equals an exact value

Scenario: a deliberate trailing blank line is addressable by index #

When #

printf 'body\n\n'

Then #

  • stdout line 2 equals an exact value

Scenario: contains treats a needle with regex metacharacters literally #

When #

printf 'price is $3.50 (approx)\n'

Then #

  • stdout contains $3.50 (approx)

Scenario: matches requires escaping a literal metacharacter #

When #

printf 'v1.2.3\n'

Then #

  • stdout matches /v1\.2\.3/

Scenario: not_matches passes when an unescaped-metachar pattern does not match #

When #

printf 'ab\n'

Then #

  • stdout does not match /a.c/

Scenario: a tab-separated record contains the exact tab byte #

When #

printf 'name\tvalue\n'

Then #

  • stdout contains "name\tvalue"

Scenario: quotes and brackets survive an exact equals #

When #

printf '[{"id":1}]\n'

Then #

  • stdout equals an exact value

Scenario: the last of many lines is selectable by index #

When #

seq 1 100

Then #

  • stdout line 100 equals an exact value

Scenario: a line selector composes with contains #

When #

printf 'alpha\nbeta-gamma\n'

Then #

  • stdout line 2 contains gamma

Scenario: a line selector composes with a regex #

When #

printf 'k=1\nk=2\n'

Then #

  • stdout line 2 matches /^k=[0-9]$/

Scenario: stderr carries the same matcher semantics as stdout #

When #

printf 'to stderr\n' 1>&2

Then #

  • stdout is empty
  • stderr equals an exact value

atago self-hosting / subject builds #

Source: test/e2e/atago/subject.atago.yaml

Scenario: the built artifact resolves by bare name from a spec #

skipped on Windows

Given #

  • Fixture file src/mytool is created.
  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture src/mytool:

#!/bin/sh
echo "mytool ${MARK:-plain}"

Fixture atago.project.yaml:

subject:
  name: mytool
  artifact: bin/mytool
  build:
    shell: true
    command: "cp src/mytool $${artifact} && chmod +x $${artifact}"

Fixture spec.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: calls the binary by name
    steps:
      - run:
          command: mytool
      - assert:
          exit_code: 0
          stdout:
            contains: "mytool plain"

When #

${atago} run spec.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: a profile swaps the build and adds environment #

skipped on Windows

Given #

  • Fixture file src/mytool is created.
  • Fixture file src/alt is created.
  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture src/mytool:

#!/bin/sh
echo "mytool ${MARK:-plain}"

Fixture src/alt:

#!/bin/sh
echo "mytool profiled"

Fixture atago.project.yaml:

subject:
  name: mytool
  artifact: bin/mytool
  build:
    shell: true
    command: "cp src/mytool $${artifact} && chmod +x $${artifact}"
profiles:
  marked:
    build:
      shell: true
      # A different build, not just different env: the profile
      # installs a second script, so a run that ignored the build
      # swap would still print "plain".
      command: "cp src/alt $${artifact} && chmod +x $${artifact}"
    env:
      MARK: "ignored-by-alt"

Fixture spec.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: sees the profile environment
    steps:
      - run:
          command: mytool
      - assert:
          stdout:
            contains: "mytool profiled"

When #

${atago} run --profile marked spec.atago.yaml
${atago} run spec.atago.yaml

Then #

  • after ${atago} run --profile marked spec.atago.yaml:
    • exit code is 0
  • after ${atago} run spec.atago.yaml:
    • exit code is 1

Scenario: an unknown profile names the ones that exist #

skipped on Windows

Given #

  • Fixture file src/mytool is created.
  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture src/mytool:

#!/bin/sh
echo ok

Fixture atago.project.yaml:

subject:
  name: mytool
  artifact: bin/mytool
  build:
    shell: true
    command: "cp src/mytool $${artifact} && chmod +x $${artifact}"
profiles:
  cover:
    env:
      GOCOVERDIR: "/tmp"

Fixture spec.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: one
    steps:
      - run:
          command: mytool

When #

${atago} run --profile race spec.atago.yaml

Then #

  • exit code is 4
  • stderr contains declares no profile "race", cover

Scenario: an artifact nothing can execute is refused #

skipped on Windows

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

subject:
  name: mytool
  artifact: bin/mytool
  build:
    shell: true
    command: "echo not-a-program > $${artifact}"

Fixture spec.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: one
    steps:
      - run:
          command: mytool

When #

${atago} run spec.atago.yaml

Then #

  • exit code is 4
  • stderr contains execute bit

Scenario: a failing build stops the run and carries the build output #

skipped on Windows

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

subject:
  name: mytool
  artifact: bin/mytool
  build:
    shell: true
    command: "echo BUILD-BROKE >&2; exit 3"

Fixture spec.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: never runs
    steps:
      - run:
          command: mytool

When #

${atago} run spec.atago.yaml

Then #

  • exit code is 4
  • stdout does not contain never runs
  • stderr contains BUILD-BROKE, exit 3

Scenario: a build that writes nothing is caught, not passed on #

skipped on Windows

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

subject:
  name: mytool
  artifact: bin/mytool
  build:
    command: "true"

Fixture spec.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: one
    steps:
      - run:
          command: mytool

When #

${atago} run spec.atago.yaml

Then #

  • exit code is 4
  • stderr contains produced no file

Scenario: a manifest with no subject leaves the run untouched #

Given #

  • Fixture file atago.project.yaml is created.
  • Fixture file spec.atago.yaml is created.

Inputs #

Fixture atago.project.yaml:

env:
  JUST_ENV: "1"

Fixture spec.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: one
    steps:
      - run:
          command: echo hi
      - assert:
          exit_code: 0

When #

${atago} run spec.atago.yaml

Then #

  • exit code is 0
  • stderr does not contain building

atago self-hosting / suite env from setup #

Source: test/e2e/atago/suite_env.atago.yaml

Scenario: a value captured in setup reaches every scenario as env #

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
  setup:
    - run:
        command: echo 127.0.0.1:19991
    - store:
        name: addr
        from:
          stdout:
            trim: true
  env:
    REGISTRY_ADDR: "${addr}"
scenarios:
  - name: reads the address
    steps:
      - run:
          shell: true
          command: "echo got=$REGISTRY_ADDR"
      - assert:
… (truncated, 2 more lines)

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 0

Scenario: a service publishes its ephemeral address into suite env #

Given #

  • Fixture file inner_service.atago.yaml is created.

Inputs #

Fixture inner_service.atago.yaml:

version: "1"
suite:
  name: inner service
  setup:
    - service:
        name: publisher
        command: "sh -c 'printf 127.0.0.1:19992 > ready.txt; sleep 30'"
        ready:
          file: ready.txt
          store: svc_addr
          timeout: 10s
  env:
    SERVICE_ADDR: "${svc_addr}"
scenarios:
  - name: reads the published address
    steps:
      - run:
          shell: true
          command: "echo got=$SERVICE_ADDR"
      - assert:
… (truncated, 2 more lines)

When #

${atago} run inner_service.atago.yaml

Then #

  • exit code is 0

Scenario: a setup step still runs when a later key is not resolvable yet #

Given #

  • Fixture file inner_order.atago.yaml is created.

Inputs #

Fixture inner_order.atago.yaml:

version: "1"
suite:
  name: inner order
  setup:
    # This run PRODUCES the value the env key below consumes, so at
    # this moment the key cannot resolve. Refusing here would make
    # the pattern impossible; the entry is simply not part of this
    # child's environment yet.
    - run:
        shell: true
        command: "echo produced=[$LATER] 127.0.0.1:19993"
    - store:
        name: later_addr
        from:
          stdout:
            trim: true
  env:
    LATER: "${later_addr}"
scenarios:
  - name: the value exists by the time scenarios run
… (truncated, 7 more lines)

When #

${atago} run inner_order.atago.yaml

Then #

  • exit code is 0

Scenario: an env value nothing defines is refused instead of leaking #

Given #

  • Fixture file inner_typo.atago.yaml is created.

Inputs #

Fixture inner_typo.atago.yaml:

version: "1"
suite:
  name: inner typo
  setup:
    - run:
        command: echo 127.0.0.1:19994
    - store:
        name: proxy_addr
        from:
          stdout:
            trim: true
  env:
    GOPROXY: "${proxy_url}"
scenarios:
  - name: would have received the literal text
    steps:
      - run:
          command: echo hi

When #

${atago} run inner_typo.atago.yaml

Then #

  • exit code is 4
  • stdout contains environment setup, suite.env GOPROXY references ${proxy_url}, literal text, proxy_addr

Scenario: an unset host variable in suite env stays literal, as documented #

Given #

  • Fixture file inner_hostenv.atago.yaml is created.

Inputs #

Fixture inner_hostenv.atago.yaml:

version: "1"
suite:
  name: inner hostenv
  env:
    MAYBE: "a:${env:ATAGO_DEFINITELY_UNSET_XYZ}:b"
scenarios:
  - name: an optional host variable is not an error
    steps:
      - run:
          command: echo ok
      - assert:
          exit_code: 0

When #

${atago} run inner_hostenv.atago.yaml

Then #

  • exit code is 0

Scenario: an escaped reference stays literal on purpose #

Given #

  • Fixture file inner_escaped.atago.yaml is created.

Inputs #

Fixture inner_escaped.atago.yaml:

version: "1"
suite:
  name: inner escaped
  env:
    TEMPLATE: "$$${not_a_variable}"
scenarios:
  - name: the literal survives
    steps:
      - run:
          shell: true
          command: "echo got=$TEMPLATE"
      - assert:
          stdout:
            contains: "got=$${not_a_variable}"

When #

${atago} run inner_escaped.atago.yaml

Then #

  • exit code is 0

atago self-hosting / suite setup #

Source: test/e2e/atago/suite_setup.atago.yaml

Scenario: setup runs once, shares stores and env, and teardown always runs #

skipped on Windows

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: inner
  env:
    FLAG: suite-env-flag
  setup:
    - run:
        shell: true
        command: "echo boot-42 > ${suitedir}/t.txt && cat ${suitedir}/t.txt"
    - store:
        name: bootid
        from:
          stdout:
            matches: "boot-[0-9]+"
  teardown:
    - run:
        shell: true
        command: "echo swept ${bootid}"
scenarios:
  - name: one
… (truncated, 12 more lines)

When #

${atago} run --verbose ok.atago.yaml

Then #

  • exit code is 0
  • stdout contains 2 passed

Scenario: a failing setup errors every scenario and none runs (exit 4) #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: inner
  setup:
    - run: {command: definitely-not-a-real-binary-xyz}
scenarios:
  - name: never runs
    steps:
      - run: {shell: true, command: echo unreached}
  - name: never runs either
    steps:
      - run: {shell: true, command: echo unreached}

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 4
  • stdout contains suite setup, 0 passed, 2 errored
  • stdout does not contain unreached

Scenario: a suite service starts once and its store reaches every scenario #

skipped on Windows

Given #

  • Fixture file svc.atago.yaml is created.

Inputs #

Fixture svc.atago.yaml:

version: "1"
suite:
  name: inner
  setup:
    - service:
        name: peer
        shell: true
        command: "echo addr-9999 > ${suitedir}/ready.txt && sleep 30"
        ready:
          file: "${suitedir}/ready.txt"
          store: addr
          timeout: 5s
scenarios:
  - name: dials
    steps:
      - run: {shell: true, command: "echo dial ${addr}"}
      - assert:
          stdout: {contains: dial addr-9999}

When #

${atago} run svc.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed

Scenario: a failing suite teardown is loud but does not flip the verdict #

Given #

  • Fixture file td.atago.yaml is created.

Inputs #

Fixture td.atago.yaml:

version: "1"
suite:
  name: inner
  setup:
    - run: {shell: true, command: echo fine}
  teardown:
    - run: {command: definitely-not-a-real-binary-xyz}
scenarios:
  - name: passes
    steps:
      - run: {shell: true, command: echo ok}
      - assert:
          stdout: {contains: ok}

When #

${atago} run td.atago.yaml

Then #

  • exit code is 0
  • stdout contains 1 passed, SUITE TEARDOWN FAILED

atago self-hosting / step timeouts (suite default + escape hatch) #

Source: test/e2e/atago/timeouts.atago.yaml

Scenario: suite.timeout kills a hanging step and the hint names it #

skipped on Windows

Given #

  • Fixture file hang.atago.yaml is created.

Inputs #

Fixture hang.atago.yaml:

version: "1"
suite:
  name: hang
  timeout: 1s
scenarios:
  - name: sleeps past the suite bound
    steps:
      - run:
          shell: true
          command: sleep 300
      - assert:
          exit_code: 0

When #

${atago} run hang.atago.yaml

Then #

  • exit code is 1
  • stdout contains timed out, suite.timeout

Scenario: a step timeout beats the suite timeout and the hint says run.timeout #

skipped on Windows

Given #

  • Fixture file step_wins.atago.yaml is created.

Inputs #

Fixture step_wins.atago.yaml:

version: "1"
suite:
  name: step-wins
  timeout: 30s
scenarios:
  - name: the step's own 1s bound fires first
    steps:
      - run:
          shell: true
          command: sleep 300
          timeout: 1s
      - assert:
          exit_code: 0

When #

${atago} run step_wins.atago.yaml

Then #

  • exit code is 1
  • stdout contains timed out, run.timeout

Scenario: timeout zero disables a short suite bound #

skipped on Windows

Given #

  • Fixture file optout.atago.yaml is created.

Inputs #

Fixture optout.atago.yaml:

version: "1"
suite:
  name: optout
  timeout: 1s
scenarios:
  - name: the opted-out step outlives the suite bound
    steps:
      - run:
          shell: true
          command: sleep 2
          timeout: "0"
      - assert:
          exit_code: 0

When #

${atago} run optout.atago.yaml

Then #

  • exit code is 0

Scenario: a killed step fails even when nothing asserts on it #

skipped on Windows

Given #

  • Fixture file unobserved.atago.yaml is created.

Inputs #

Fixture unobserved.atago.yaml:

version: "1"
suite:
  name: unobserved
  timeout: 1s
scenarios:
  - name: nothing looks at the killed command
    steps:
      - run:
          shell: true
          command: sleep 300

When #

${atago} run unobserved.atago.yaml

Then #

  • exit code is 1
  • stdout contains run completes before its timeout, was killed, suite.timeout

Scenario: an assert that ignores the result does not mask the kill #

skipped on Windows

Given #

  • Fixture file file_assert.atago.yaml is created.

Inputs #

Fixture file_assert.atago.yaml:

version: "1"
suite:
  name: file-assert
scenarios:
  - name: the assert reads a file the step did not produce
    steps:
      - fixture:
          file: made.txt
          content: |
            hi
      - run:
          shell: true
          command: sleep 300
          timeout: 500ms
      - assert:
          file:
            path: made.txt
            contains: hi

When #

${atago} run file_assert.atago.yaml

Then #

  • exit code is 1
  • stdout contains run completes before its timeout, run.timeout

Scenario: an assert on the killed result keeps the timeout observable #

skipped on Windows

Given #

  • Fixture file observed.atago.yaml is created.

Inputs #

Fixture observed.atago.yaml:

version: "1"
suite:
  name: observed
scenarios:
  - name: the assert inspects the timeout instead of ignoring it
    steps:
      - run:
          shell: true
          command: sleep 300
          timeout: 500ms
      - assert:
          exit_code:
            not: 0

When #

${atago} run observed.atago.yaml

Then #

  • exit code is 0

Scenario: timeout zero keeps an unasserted step green #

skipped on Windows

Given #

  • Fixture file optout_bare.atago.yaml is created.

Inputs #

Fixture optout_bare.atago.yaml:

version: "1"
suite:
  name: optout-bare
  timeout: 500ms
scenarios:
  - name: the opted-out step is never killed so nothing fails it
    steps:
      - run:
          shell: true
          command: sleep 1
          timeout: "0"

When #

${atago} run optout_bare.atago.yaml

Then #

  • exit code is 0

Scenario: an invalid suite.timeout is a load-time error #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: bad
  timeout: fast
scenarios:
  - name: never runs
    steps:
      - run:
          command: echo hi

When #

${atago} run bad.atago.yaml

Then #

  • exit code is 2
  • stderr contains suite.timeout

Scenario: a session outlived by its program says so instead of blaming the clock #

skipped on Windows

Given #

  • Fixture file outlived.atago.yaml is created.

Inputs #

Fixture outlived.atago.yaml:

version: "1"
suite: {name: outlived}
scenarios:
  - name: the session ends with cat still reading
    steps:
      - pty:
          command: cat
          timeout: 2s
          session:
            - send: "hello\n"
            - expect: "hello"

When #

${atago} run outlived.atago.yaml

Then #

  • exit code is 1
  • stdout contains the program was still running, send its quit key as the last action
  • stdout does not contain raise the timeout if the command is merely slow

Scenario: an expect that never matches still reports the pattern, not the quit advice #

skipped on Windows

Given #

  • Fixture file nomatch.atago.yaml is created.

Inputs #

Fixture nomatch.atago.yaml:

version: "1"
suite: {name: nomatch}
scenarios:
  - name: the prompt never appears
    steps:
      - pty:
          command: cat
          timeout: 2s
          session:
            - expect: "never-going-to-appear"

When #

${atago} run nomatch.atago.yaml

Then #

  • exit code is 1
  • stdout contains never-going-to-appear, never appeared in the terminal transcript
  • stdout does not contain send its quit key

Scenario: a timeout that expires before the process starts is still a timeout #

Given #

  • Fixture file prestart.atago.yaml is created.

Inputs #

Fixture prestart.atago.yaml:

version: "1"
suite:
  name: prestart
scenarios:
  - name: the budget is spent before the process exists
    steps:
      - run:
          shell: true
          command: echo hi
          timeout: 1ns
      - assert:
          exit_code: 0

When #

${atago} run prestart.atago.yaml

Then #

  • exit code is 1
  • stdout contains timed out, run.timeout, does not contain failed to execute

atago self-hosting / tui #

Source: test/e2e/atago/tui.atago.yaml

Scenario: a pty step exports a usable TERM by default #

skipped on Windows

When #

# interactive (pty): echo "TERM=[$TERM]"

Then #

  • exit code is 0
  • stdout contains TERM=[xterm-256color]

Scenario: an explicit TERM overrides the default #

skipped on Windows

When #

# interactive (pty): echo "TERM=[$TERM]"

Then #

  • exit code is 0
  • stdout contains TERM=[vt100]

Scenario: an expect does not re-match a consumed pattern #

skipped on Windows

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: stale expect must not re-match
    steps:
      - pty:
          shell: true
          command: 'printf "AAA\n"'
          timeout: 1s
          session:
            - expect: "AAA"
            - expect: "AAA"

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains 1 failed

Scenario: a failing screen assert frames a CJK screen squarely #

skipped on Windows

Given #

  • Fixture file inner.atago.yaml is created.

Inputs #

Fixture inner.atago.yaml:

version: "1"
suite:
  name: inner
scenarios:
  - name: screen assert that must fail
    steps:
      - pty:
          shell: true
          command: 'printf "abcdefgh\r\n日本語メニュー\r\n"'
          rows: 4
          cols: 20
      - assert:
          screen:
            contains: "NOT ON THIS SCREEN"

When #

${atago} run inner.atago.yaml

Then #

  • exit code is 1
  • stdout contains | abcdefgh |, | 日本語メニュー |

Scenario: a screen assert sees wide characters at their true width #

skipped on Windows

When #

# interactive (pty): printf "\033[2J\033[1;1H日本語\033[1;7H[OK]"

Then #

  • rendered screen contains 日本語[OK], does not contain 日本語 [OK]

Scenario: less -X renders a real pager onto the screen #

only when command -v less succeeds · skipped on Windows

Given #

  • Fixture file page.txt is created.

Inputs #

Fixture page.txt:

Alpha line
Beta line
Gamma line

When #

# interactive (pty): less -X page.txt

Then #

  • rendered screen contains Alpha line
  • rendered screen contains Gamma line

atago self-hosting / variable resolution semantics #

Source: test/e2e/atago/var_resolution.atago.yaml

Scenario: a doubled dollar keeps the braces literal #

skipped on Windows

When #

echo pre-$${keep}-post

Then #

  • stdout contains ${keep}

Scenario: the workdir builtin expands to the scenario directory #

When #

echo at=${workdir}

Then #

  • stdout does not contain $${workdir}

Scenario: the atago builtin resolves to the binary under test #

When #

${atago} --version

Then #

  • exit code is 0
  • stdout contains atago

Scenario: an env reference expands from the host environment #

When #

echo path=${env:PATH}

Then #

  • stdout matches /path=.+/

Scenario: shell true defers an unknown reference to the shell #

When #

echo [${undefined_in_atago}]

Then #

  • exit code is 0
  • stdout equals an exact value

Scenario: an unresolved variable is a hard error, not a silent empty #

Given #

  • Fixture file typo.atago.yaml is created.

Inputs #

Fixture typo.atago.yaml:

version: "1"
suite: {name: typo}
scenarios:
  - name: a misspelled variable stops the run
    steps:
      - run: {command: "echo ${reuslt}"}

When #

${atago} run typo.atago.yaml

Then #

  • exit code is 4
  • stdout contains references ${reuslt}, no variable with that name is defined

Scenario: an unset env reference names the missing variable #

Given #

  • Fixture file unsetenv.atago.yaml is created.

Inputs #

Fixture unsetenv.atago.yaml:

version: "1"
suite: {name: unsetenv}
scenarios:
  - name: an unset env var stops the run
    steps:
      - run: {command: "echo ${env:ATAGO_SURELY_UNSET_VAR}"}

When #

${atago} run unsetenv.atago.yaml

Then #

  • exit code is 4
  • stdout contains environment variable ATAGO_SURELY_UNSET_VAR is not set

Scenario: the same rule covers the command that starts a pty session #

skipped on Windows

Given #

  • Fixture file ptytypo.atago.yaml is created.

Inputs #

Fixture ptytypo.atago.yaml:

version: "1"
suite: {name: ptytypo}
scenarios:
  - name: a misspelled variable in a pty command stops the run
    steps:
      - pty:
          command: "cat $${no_such_file}"
          timeout: 10s
          session:
            - send: {key: ctrl-d}

When #

${atago} run ptytypo.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4505, pty.command references ${no_such_file}

Scenario: an unresolved reference reports one diagnostic code everywhere #

Given #

  • Fixture file coded.atago.yaml is created.

Inputs #

Fixture coded.atago.yaml:

version: "1"
suite: {name: coded}
scenarios:
  - name: a run step
    steps:
      - run: {command: "echo $${no_such_var}"}

When #

${atago} run coded.atago.yaml

Then #

  • exit code is 4
  • stdout contains ATG4505, run.command references ${no_such_var}

atago self-hosting / verbose #

Source: test/e2e/atago/verbose.atago.yaml

Scenario: verbose shows a passing scenario’s command, output, and verdicts #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: greets
    steps:
      - run:
          shell: true
          command: echo hello-trace
      - assert:
          exit_code: 0
          stdout:
            contains: hello-trace

When #

${atago} run --verbose ok.atago.yaml

Then #

  • exit code is 0
  • stdout contains sample / greets, echo hello-trace, exit 0, ok assert

Scenario: without –verbose the trace is absent #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: greets
    steps:
      - run:
          shell: true
          command: echo hello-trace
      - assert:
          exit_code: 0

When #

${atago} run ok.atago.yaml

Then #

  • exit code is 0
  • stdout does not contain echo hello-trace

Scenario: verbose with a JSON report keeps stdout pure and traces to stderr #

Given #

  • Fixture file ok.atago.yaml is created.

Inputs #

Fixture ok.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: greets
    steps:
      - run:
          shell: true
          command: echo hello-trace
      - assert:
          exit_code: 0

When #

${atago} run --verbose --report json ok.atago.yaml

Then #

  • exit code is 0
  • stdout at $.schema_version equals 1
  • stderr contains exit 0

Scenario: a failing run under –verbose renders the FAILED block exactly once #

Given #

  • Fixture file bad.atago.yaml is created.

Inputs #

Fixture bad.atago.yaml:

version: "1"
suite:
  name: sample
scenarios:
  - name: mismatch
    steps:
      - run:
          shell: true
          command: echo hello
      - assert:
          stdout:
            contains: goodbye

When #

${atago} run --verbose bad.atago.yaml

Then #

  • exit code is 1
  • stdout contains FAIL assert, FAILED:
  • stdout does not match /(?s)FAILED:.*FAILED:/

atago self-hosting / version #

Source: test/e2e/atago/version.atago.yaml

Scenario: version command prints the binary name #

When #

${atago} version

Then #

  • exit code is 0
  • stdout contains atago
  • stderr is empty

Scenario: unknown command is a configuration error #

When #

${atago} frobnicate

Then #

  • exit code is 3
  • stderr contains unknown command

atago self-hosting / yaml stream matcher #

Source: test/e2e/atago/yaml.atago.yaml

Scenario: a yaml stream matcher selects and asserts a decoded value (#9) #

Given #

  • Fixture file yaml_ok.atago.yaml is created.

Inputs #

Fixture yaml_ok.atago.yaml:

version: "1"
suite:
  name: yaml-matcher
scenarios:
  - name: command emits yaml and the matcher reads it
    steps:
      - fixture:
          file: data.yaml
          content: |
            name: alice
            items:
              - id: 1
              - id: 2
      - run:
          command: cat data.yaml
      - assert:
          stdout:
            yaml:
              path: "$.name"
              equals: alice
… (truncated, 5 more lines)

When #

${atago} run yaml_ok.atago.yaml

Then #

  • exit code is 0
  • stdout contains PASS
  • stdout does not contain matcher not supported yet

Scenario: a yaml matcher mismatch fails the inner spec (#9) #

Given #

  • Fixture file yaml_fail.atago.yaml is created.

Inputs #

Fixture yaml_fail.atago.yaml:

version: "1"
suite:
  name: yaml-matcher-fail
scenarios:
  - name: wrong expected value
    steps:
      - fixture:
          file: data.yaml
          content: |
            name: alice
      - run:
          command: cat data.yaml
      - assert:
          stdout:
            yaml:
              path: "$.name"
              equals: bob

When #

${atago} run yaml_fail.atago.yaml

Then #

  • exit code is 1
  • stdout contains FAILED