luis/ob-elixir
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs and tasks

Browse Source
This commit is contained in:
Luis Bueso
2026-01-24 13:44:20 +01:00
commit 3ca9c10678
16 changed files with 6704 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

626
docs/02-testing-emacs-elisp.md Normal file
View File

@@ -0,0 +1,626 @@
# Testing Emacs Lisp Packages
This document covers testing strategies and tools for Emacs Lisp packages, with focus on org-babel language implementations.
## Table of Contents
- [Testing Frameworks Overview](#testing-frameworks-overview)
- [ERT (Emacs Lisp Regression Testing)](#ert-emacs-lisp-regression-testing)
- [Buttercup](#buttercup)
- [Test Organization](#test-organization)
- [Testing org-babel Specifics](#testing-org-babel-specifics)
- [Mocking and Stubbing](#mocking-and-stubbing)
- [CI/CD Integration](#cicd-integration)
- [Build Tools](#build-tools)
- [References](#references)
---
## Testing Frameworks Overview
| Framework | Style | Best For | Included |
|---------------|------------------|-------------------------|------------------------|
| **ERT** | xUnit | Unit tests, built-in | Yes, in Emacs |
| **Buttercup** | BDD/RSpec | Readable specs, mocking | No, install from MELPA |
| **Ecukes** | Cucumber/Gherkin | Integration/acceptance | No, install from MELPA |
For a package like ob-elixir, **ERT** is recommended as it:
- Comes built into Emacs
- Is well-documented
- Works well with org-mode's own test patterns
- Integrates easily with CI
---
## ERT (Emacs Lisp Regression Testing)
### Basic Test Structure
```elisp
;;; test-ob-elixir.el --- Tests for ob-elixir -*- lexical-binding: t; -*-
(require 'ert)
(require 'ob-elixir)
;; Simple test
(ert-deftest ob-elixir-test-basic-evaluation ()
"Test basic Elixir code evaluation."
(should (equal "2" (ob-elixir-evaluate "1 + 1" '()))))
;; Test with setup
(ert-deftest ob-elixir-test-variable-injection ()
"Test that variables are properly injected."
(let ((params '((:var . ("x" . 5)))))
(should (equal "10"
(ob-elixir-evaluate "x * 2" params)))))
;; Test for expected errors
(ert-deftest ob-elixir-test-syntax-error ()
"Test that syntax errors are handled."
(should-error (ob-elixir-evaluate "def foo(" '())
:type 'ob-elixir-execution-error))
;; Test with negation
(ert-deftest ob-elixir-test-not-nil ()
"Test result is not nil."
(should-not (null (ob-elixir-evaluate ":ok" '()))))
```
### ERT Assertions
```elisp
;; Basic assertions
(should FORM) ; FORM should be non-nil
(should-not FORM) ; FORM should be nil
(should-error FORM) ; FORM should signal an error
(should-error FORM :type 'error-type) ; Specific error type
;; Equality checks
(should (equal expected actual))
(should (string= "expected" actual))
(should (= 42 actual)) ; Numeric equality
(should (eq 'symbol actual)) ; Symbol identity
(should (eql 1.0 actual)) ; Numeric with type
;; Pattern matching
(should (string-match-p "pattern" actual))
(should (cl-every #'integerp list))
;; With custom messages (ERT 28+)
(ert-deftest example-with-message ()
(let ((result (some-function)))
(should (equal expected result)
(format "Expected %s but got %s" expected result))))
```
### Test Fixtures
```elisp
;; Using let for setup
(ert-deftest ob-elixir-test-with-fixture ()
(let ((org-babel-default-header-args:elixir '((:results . "value")))
(test-code "Enum.sum([1, 2, 3])"))
(should (equal "6" (ob-elixir-evaluate test-code '())))))
;; Shared setup with macros
(defmacro ob-elixir-test-with-temp-buffer (&rest body)
"Execute BODY in a temporary org buffer."
`(with-temp-buffer
(org-mode)
,@body))
(ert-deftest ob-elixir-test-in-org-buffer ()
(ob-elixir-test-with-temp-buffer
(insert "#+BEGIN_SRC elixir\n1 + 1\n#+END_SRC")
(goto-char (point-min))
(forward-line 1)
(should (equal "2" (org-babel-execute-src-block)))))
;; Setup/teardown pattern
(defvar ob-elixir-test--saved-command nil)
(defun ob-elixir-test-setup ()
"Setup for ob-elixir tests."
(setq ob-elixir-test--saved-command ob-elixir-command))
(defun ob-elixir-test-teardown ()
"Teardown for ob-elixir tests."
(setq ob-elixir-command ob-elixir-test--saved-command))
(ert-deftest ob-elixir-test-with-setup-teardown ()
(ob-elixir-test-setup)
(unwind-protect
(progn
(setq ob-elixir-command "/custom/path/elixir")
(should (string= "/custom/path/elixir" ob-elixir-command)))
(ob-elixir-test-teardown)))
```
### Running ERT Tests
```elisp
;; Interactive (in Emacs)
M-x ert RET t RET ; Run all tests
M-x ert RET ob-elixir RET ; Run tests matching "ob-elixir"
M-x ert-run-tests-interactively
;; From command line
emacs -batch -l ert -l ob-elixir.el -l test-ob-elixir.el \
-f ert-run-tests-batch-and-exit
;; With specific selector
emacs -batch -l ert -l test-ob-elixir.el \
--eval "(ert-run-tests-batch-and-exit 'ob-elixir-test-basic)"
```
---
## Buttercup
Buttercup provides RSpec-style BDD testing with better mocking support.
### Installation
```elisp
;; In Cask file or package installation
(depends-on "buttercup")
;; Or via use-package
(use-package buttercup :ensure t)
```
### Basic Structure
```elisp
;;; test-ob-elixir-buttercup.el --- Buttercup tests -*- lexical-binding: t; -*-
(require 'buttercup)
(require 'ob-elixir)
(describe "ob-elixir"
(describe "basic evaluation"
(it "evaluates simple arithmetic"
(expect (ob-elixir-evaluate "1 + 1" '())
:to-equal "2"))
(it "returns nil for empty input"
(expect (ob-elixir-evaluate "" '())
:to-be nil)))
(describe "variable injection"
(before-each
(setq test-params '((:var . ("x" . 10)))))
(it "injects variables into code"
(expect (ob-elixir-evaluate "x * 2" test-params)
:to-equal "20"))
(it "handles multiple variables"
(setq test-params '((:var . ("x" . 10))
(:var . ("y" . 5))))
(expect (ob-elixir-evaluate "x + y" test-params)
:to-equal "15")))
(describe "error handling"
(it "signals error on syntax error"
(expect (ob-elixir-evaluate "def foo(" '())
:to-throw 'ob-elixir-execution-error))))
```
### Buttercup Matchers
```elisp
;; Equality
(expect value :to-equal expected)
(expect value :to-be expected) ; eq comparison
(expect value :to-be-truthy)
(expect value :to-be nil)
;; Comparison
(expect value :to-be-greater-than 5)
(expect value :to-be-less-than 10)
;; Strings
(expect string :to-match "regexp")
(expect string :to-contain-string "substring")
;; Errors
(expect (lambda () (error-func)) :to-throw)
(expect (lambda () (error-func)) :to-throw 'error-type)
;; Spies (see Mocking section)
(expect 'function :to-have-been-called)
(expect 'function :to-have-been-called-with args)
(expect 'function :to-have-been-called-times n)
```
### Running Buttercup
```bash
# With Cask
cask exec buttercup -L .
# With Eldev
eldev test buttercup
# From command line directly
emacs -batch -L . -l buttercup \
-f buttercup-run-discover
```
---
## Test Organization
### Directory Structure
```
ob-elixir/
├── ob-elixir.el
├── test/
│ ├── test-ob-elixir.el # Main test file
│ ├── test-ob-elixir-session.el # Session-specific tests
│ ├── test-ob-elixir-vars.el # Variable handling tests
│ └── resources/
│ ├── sample.org # Test org files
│ └── test-project/ # Test Mix project
├── Cask # Or Eldev file
└── Makefile
```
### Test File Template
```elisp
;;; test-ob-elixir.el --- Tests for ob-elixir -*- lexical-binding: t; -*-
;;; Commentary:
;; Tests for ob-elixir package.
;; Run with: make test
;; Or: emacs -batch -l ert -l test-ob-elixir.el -f ert-run-tests-batch-and-exit
;;; Code:
(require 'ert)
;; Add source directory to load path
(let ((dir (file-name-directory (or load-file-name buffer-file-name))))
(add-to-list 'load-path (expand-file-name ".." dir)))
(require 'ob-elixir)
;;; Test Helpers
(defun ob-elixir-test-org-block (code &optional header-args)
"Create org src block with CODE and evaluate it."
(with-temp-buffer
(org-mode)
(insert (format "#+BEGIN_SRC elixir %s\n%s\n#+END_SRC"
(or header-args "")
code))
(goto-char (point-min))
(forward-line 1)
(org-babel-execute-src-block)))
;;; Unit Tests
(ert-deftest ob-elixir-test-command-exists ()
"Test that Elixir command is accessible."
(should (executable-find ob-elixir-command)))
(ert-deftest ob-elixir-test-simple-evaluation ()
"Test simple code evaluation."
(should (equal "2" (ob-elixir-test-org-block "1 + 1"))))
;;; Integration Tests
(ert-deftest ob-elixir-integration-full-block ()
"Test full org-babel workflow."
(skip-unless (executable-find "elixir"))
(should (equal "6" (ob-elixir-test-org-block "Enum.sum([1, 2, 3])"))))
(provide 'test-ob-elixir)
;;; test-ob-elixir.el ends here
```
---
## Testing org-babel Specifics
### Testing with Real Org Buffers
```elisp
(ert-deftest ob-elixir-test-full-org-workflow ()
"Test the complete org-babel evaluation workflow."
(skip-unless (executable-find "elixir"))
(with-temp-buffer
(org-mode)
(insert "#+NAME: test-block
#+BEGIN_SRC elixir :results value
Enum.map([1, 2, 3], &(&1 * 2))
#+END_SRC")
(goto-char (point-min))
(org-babel-goto-named-src-block "test-block")
(let ((result (org-babel-execute-src-block)))
(should (equal '(2 4 6) result)))))
```
### Testing Variable Passing
```elisp
(ert-deftest ob-elixir-test-var-passing ()
"Test :var header argument handling."
(with-temp-buffer
(org-mode)
(insert "#+BEGIN_SRC elixir :var x=5 :var y=10
x + y
#+END_SRC")
(goto-char (point-min))
(forward-line 1)
(should (equal "15" (org-babel-execute-src-block)))))
```
### Testing Table Input/Output
```elisp
(ert-deftest ob-elixir-test-table-input ()
"Test that tables are passed correctly."
(with-temp-buffer
(org-mode)
(insert "#+NAME: test-data
| a | 1 |
| b | 2 |
#+BEGIN_SRC elixir :var data=test-data
Enum.map(data, fn [k, v] -> {k, v * 2} end)
#+END_SRC")
(goto-char (point-min))
(search-forward "BEGIN_SRC")
(let ((result (org-babel-execute-src-block)))
(should (equal '(("a" 2) ("b" 4)) result)))))
```
### Testing Session Mode
```elisp
(ert-deftest ob-elixir-test-session-persistence ()
"Test that session maintains state between evaluations."
(skip-unless (executable-find "iex"))
(unwind-protect
(progn
;; First block: define variable
(with-temp-buffer
(org-mode)
(insert "#+BEGIN_SRC elixir :session test-session
x = 42
#+END_SRC")
(goto-char (point-min))
(forward-line 1)
(org-babel-execute-src-block))
;; Second block: use variable
(with-temp-buffer
(org-mode)
(insert "#+BEGIN_SRC elixir :session test-session
x * 2
#+END_SRC")
(goto-char (point-min))
(forward-line 1)
(should (equal "84" (org-babel-execute-src-block)))))
;; Cleanup
(when-let ((buf (get-buffer "*elixir-test-session*")))
(kill-buffer buf))))
```
---
## Mocking and Stubbing
### ERT with cl-letf
```elisp
(require 'cl-lib)
(ert-deftest ob-elixir-test-mock-shell-command ()
"Test with mocked shell command."
(cl-letf (((symbol-function 'shell-command-to-string)
(lambda (cmd) "42")))
(should (equal "42" (ob-elixir-evaluate "anything" '())))))
(ert-deftest ob-elixir-test-mock-process ()
"Test with mocked external process."
(cl-letf (((symbol-function 'call-process)
(lambda (&rest _args) 0))
((symbol-function 'org-babel-eval)
(lambda (_cmd body) (format "result: %s" body))))
(should (string-match "result:" (ob-elixir-evaluate "code" '())))))
```
### Buttercup Spies
```elisp
(describe "ob-elixir with mocks"
(before-each
(spy-on 'shell-command-to-string :and-return-value "mocked-result")
(spy-on 'message))
(it "uses shell command"
(ob-elixir-evaluate "code" '())
(expect 'shell-command-to-string :to-have-been-called))
(it "passes correct arguments"
(ob-elixir-evaluate "1 + 1" '())
(expect 'shell-command-to-string
:to-have-been-called-with
(string-match-p "elixir" (spy-calls-args-for
'shell-command-to-string 0))))
(it "can call through to original"
(spy-on 'some-func :and-call-through)
;; This will call the real function but track calls
))
```
### Testing Without External Dependencies
```elisp
(ert-deftest ob-elixir-test-without-elixir ()
"Test behavior when Elixir is not installed."
(cl-letf (((symbol-function 'executable-find)
(lambda (_) nil)))
(should-error (ob-elixir-check-installation)
:type 'user-error)))
```
---
## CI/CD Integration
### GitHub Actions Workflow
```yaml
# .github/workflows/test.yml
name: Tests
on:
push:
branches: [main, master]
pull_request:
branches: [main, master]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
emacs-version: ['27.2', '28.2', '29.1']
steps:
- uses: actions/checkout@v4
- name: Set up Emacs
uses: purcell/setup-emacs@master
with:
version: ${{ matrix.emacs-version }}
- name: Set up Elixir
uses: erlef/setup-beam@v1
with:
elixir-version: '1.15'
otp-version: '26'
- name: Install Eldev
run: curl -fsSL https://raw.github.com/doublep/eldev/master/webinstall/github-eldev | sh
- name: Run tests
run: eldev test
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Emacs
uses: purcell/setup-emacs@master
with:
version: '29.1'
- name: Install Eldev
run: curl -fsSL https://raw.github.com/doublep/eldev/master/webinstall/github-eldev | sh
- name: Lint
run: eldev lint
```
### Makefile
```makefile
EMACS ?= emacs
BATCH = $(EMACS) -Q -batch -L .
.PHONY: test test-ert compile lint clean
all: compile test
compile:
$(BATCH) -f batch-byte-compile ob-elixir.el
test: test-ert
test-ert:
$(BATCH) -l ert -l test/test-ob-elixir.el \
-f ert-run-tests-batch-and-exit
lint:
$(BATCH) -l package-lint \
--eval "(setq package-lint-main-file \"ob-elixir.el\")" \
-f package-lint-batch-and-exit ob-elixir.el
clean:
rm -f *.elc test/*.elc
```
---
## Build Tools
### Eldev (Recommended)
```elisp
; Eldev file
(eldev-use-package-archive 'gnu)
(eldev-use-package-archive 'melpa)
; Dependencies
(eldev-add-extra-dependencies 'test 'buttercup)
; Test configuration
(setf eldev-test-framework 'ert) ; or 'buttercup
; Lint configuration
(setf eldev-lint-default '(elisp package))
```
```bash
# Common Eldev commands
eldev test # Run tests
eldev lint # Run linters
eldev compile # Byte-compile
eldev clean # Clean build artifacts
eldev doctor # Check project health
```
### Cask
```elisp
; Cask file
(source gnu)
(source melpa)
(package-file "ob-elixir.el")
(development
(depends-on "ert")
(depends-on "buttercup")
(depends-on "package-lint"))
```
```bash
# Common Cask commands
cask install # Install dependencies
cask exec ert-runner # Run ERT tests
cask exec buttercup -L . # Run Buttercup tests
cask build # Byte-compile
```
---
## References
- [ERT Manual](https://www.gnu.org/software/emacs/manual/html_node/ert/)
- [Buttercup Documentation](https://github.com/jorgenschaefer/emacs-buttercup)
- [Eldev Documentation](https://github.com/doublep/eldev)
- [Cask Documentation](https://github.com/cask/cask)
- [GitHub Actions for Emacs](https://github.com/purcell/setup-emacs)
- [Org Mode Test Suite](https://git.savannah.gnu.org/cgit/emacs/org-mode.git/tree/testing)
- [Package-lint](https://github.com/purcell/package-lint)