main.go
// Command mirror is a voice-reactive mirror: your camera, warped every frame by
// what the microphone hears.
//
// The app itself lives in ./ui so the same tree can be built as a desktop
// command and bound into a mobile host — gomobile cannot bind package main, so
// the CLI generates a bind package from ui.Root and ui.Config.
package main
import (
"log"
"github.com/doug/gophics/app"
"github.com/doug/gophics/examples/mirror/ui"
)
func main() {
if err := app.Run(ui.Root(), ui.Config()); err != nil {
log.Fatal(err)
}
}
ui/effect.go
package ui
import (
"image"
"math"
)
// The effect is deliberately a pure function over two pixel buffers and a few
// numbers: no camera, no microphone, no widgets. That is what makes it testable
// without hardware — the tests below drive it with a synthetic frame and
// synthetic audio — and it is also what keeps it fast, because the whole thing
// can be reduced to table lookups before the pixel loop starts.
// Params is one frame's worth of modulation.
type Params struct {
// Level is the microphone's current level, 0..1.
Level float32
// Bands is the input spectrum, low frequency first, 0..1. Any length.
Bands []float32
// T is a monotonically rising time in seconds, for motion that continues
// while the room is quiet.
T float32
// Amount scales the whole effect, 0..1 — the one control the UI exposes.
Amount float32
// Mirror flips horizontally. A front camera shows you un-mirrored, which
// reads as somebody else's face rather than your own reflection.
Mirror bool
}
// Warp writes a voice-modulated version of src into dst. dst and src must be
// the same size; Warp is a no-op otherwise rather than a panic, because the
// camera can change frame size underneath a running preview.
//
// The two displacements are separable — the horizontal one depends only on the
// row and the vertical one only on the column — so both collapse into lookup
// tables computed once per frame. The pixel loop is then four loads and a
// clamp, which is what lets this run per frame in Go rather than needing a
// shader.
func Warp(dst, src *image.RGBA, p Params) {
b := src.Bounds()
if dst.Bounds() != b || b.Empty() {
return
}
w, h := b.Dx(), b.Dy()
amt := clamp01(p.Amount)
level := clamp01(p.Level)
// Columns rise with the energy in their part of the spectrum, so the image
// behaves like a bar chart made of your own face.
colDY := make([]int, w)
lift := amt * float32(h) * 0.10
for x := range w {
colDY[x] = int(bandAt(p.Bands, x, w) * lift)
}
// Rows slide sideways on a travelling wave whose amplitude is loudness, so
// a quiet room is still and a shout ripples.
rowDX := make([]int, h)
sway := amt * level * float32(w) * 0.035
for y := range h {
rowDX[y] = int(sway * float32(math.Sin(float64(y)*0.055+float64(p.T)*3.2)))
}
// A colour split that opens up as you get louder: red and blue sampled a
// few pixels either side of green.
chroma := int(amt * level * float32(w) * 0.012)
sp, dp := src.Pix, dst.Pix
ss, ds := src.Stride, dst.Stride
for y := range h {
dx := rowDX[y]
row := y * ds
for x := range w {
sx := x
if p.Mirror {
sx = w - 1 - x
}
sx += dx
sy := y + colDY[x]
g := clampi(sx, 0, w-1)
gy := clampi(sy, 0, h-1)
base := gy*ss + g*4
o := row + x*4
dp[o+1] = sp[base+1] // green stays put; it carries the luminance
dp[o+3] = 255
if chroma == 0 {
dp[o] = sp[base]
dp[o+2] = sp[base+2]
continue
}
r := gy*ss + clampi(sx+chroma, 0, w-1)*4
bl := gy*ss + clampi(sx-chroma, 0, w-1)*4
dp[o] = sp[r]
dp[o+2] = sp[bl+2]
}
}
}
// bandAt reads the spectrum at a column, interpolating between neighbouring
// bands and squaring the result.
//
// Both parts matter. Four dozen bands across a 640-pixel frame is one band
// every thirteen columns, so picking the nearest cuts the silhouette into a
// staircase; interpolating makes the lift continuous. And squaring keeps the
// noise floor near zero — without it every band sits slightly above silence and
// the whole image shimmers in an empty room.
func bandAt(bands []float32, x, w int) float32 {
n := len(bands)
if n == 0 || w <= 0 {
return 0
}
p := (float32(x)+0.5)*float32(n)/float32(w) - 0.5
i := int(math.Floor(float64(p)))
f := p - float32(i)
a := clamp01(bands[clampi(i, 0, n-1)])
b := clamp01(bands[clampi(i+1, 0, n-1)])
v := a + (b-a)*f
return v * v
}
func clamp01(v float32) float32 {
if v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}
func clampi(v, lo, hi int) int {
if v < lo {
return lo
}
if v > hi {
return hi
}
return v
}
ui/mirror.go
// Package ui is a voice-reactive mirror: your camera, warped every frame by
// what the microphone hears. Columns of the image rise with the energy in their
// part of the spectrum, rows slide on a wave that grows with loudness, and the
// colour channels split apart as you get louder — so the picture sings.
//
// It is the driver example for the live-capture capabilities, shell.CameraPreview
// and shell.Microphone: streaming capture, as opposed to the one-shot still and
// clip that shell.Camera and shell.Audio provide. Frames arrive as *image.RGBA,
// the warp is a plain Go pixel loop (effect.go, pure and unit-tested), and the
// result is handed to one widget.Canvas. No shader, no platform image pipeline.
//
// Live capture is implemented by the web shell today; the native shells leave
// it nil and this app hides the affordance and says why, which is what every
// capability is supposed to do where a platform doesn't provide it.
//
// gophics dev -p web ./examples/mirror
package ui
import (
"fmt"
"image"
"time"
"golang.org/x/image/font/gofont/gobold"
"golang.org/x/image/font/gofont/gomono"
"golang.org/x/image/font/gofont/goregular"
"github.com/doug/gophics/app"
"github.com/doug/gophics/geom"
"github.com/doug/gophics/layout"
"github.com/doug/gophics/paint"
"github.com/doug/gophics/shell"
"github.com/doug/gophics/theme"
"github.com/doug/gophics/widget"
)
// numBands is how many spectrum bands the effect and the meter use. The
// Microphone contract folds the analyser's own resolution onto whatever length
// is asked for, so this is purely a question of how coarse the columns look.
const numBands = 48
var (
bg = paint.RGB(0.05, 0.05, 0.07)
dim = paint.Color{R: 1, G: 1, B: 1, A: 0.55}
barCol = paint.RGB(0.45, 0.86, 0.98)
)
type App struct{}
func (App) CreateState() widget.State { return &mirror{} }
type mirror struct {
widget.StateBase[App]
ctx widget.Ctx
frames shell.Frames
mon shell.Monitor
source Source // a test-installed source, in place of the camera
bands []float32
level float32
smooth float32 // level, low-passed, so the image doesn't flicker on consonants
t float32
// Output is double-buffered for the same reason the shell rotates frame
// buffers: the scene compares images by identity, so handing the canvas the
// same *image.RGBA with new pixels in it would never repaint.
out [2]*image.RGBA
cur int
show *image.RGBA
amount float32
mirrored bool
starting bool
autoTried bool // the launch-time open has been attempted, once
err string
fps float32
lastWarp time.Duration
}
// Source lets a test drive the app with frames and audio of its own. When one
// is installed the capability path is skipped entirely, so the whole app — not
// just the effect — is exercisable headless.
type Source interface {
Frame() *image.RGBA
Level() float32
Bands(dst []float32) int
}
var (
// testSource, if set, replaces the platform capabilities on mount.
testSource Source
// stateHook, if set, receives the state on mount — for tests to drive input.
stateHook func(*mirror)
)
func (m *mirror) Init(ctx widget.Ctx) {
m.ctx = ctx
m.bands = make([]float32, numBands)
m.amount = 0.75
m.mirrored = true
if testSource != nil {
m.source = testSource
}
ctx.AddTicker(m)
if stateHook != nil {
stateHook(m)
}
}
// autostart opens the camera on launch when it can do so without prompting.
//
// A demo whose whole subject is the camera should show the camera, not a button
// that shows the camera. But opening one raises a permission prompt the first
// time and lights the capture indicator, and doing that unasked the instant an
// app launches is what makes people distrust an app. So it starts immediately
// where access is already granted, and keeps the button for the case where
// starting would prompt — the prompt then follows a press, which is the only
// moment the user has asked for it.
//
// Driven from the ticker rather than Init because capabilities are not wired
// when Init runs: the tree mounts inside newCore before any shell exists, and
// they arrive on the first frame (see app/present.go). Reading them in Init
// sees nil on every platform, which is what the first version of this did.
func (m *mirror) autostart() {
if m.autoTried || m.starting || m.running() {
return
}
cam := m.ctx.CameraPreview()
if cam == nil || !m.live() {
return // capabilities not wired yet, or this platform has none
}
m.autoTried = true
cam.Authorize(func(p shell.Permission) {
if p == shell.PermissionGranted {
m.start()
}
})
}
// Dispose releases the camera and microphone. Without this the capture light
// stays on after the app closes, which is the kind of bug a user notices and
// does not forgive.
func (m *mirror) Dispose() { m.stop() }
func (m *mirror) stop() {
if m.frames != nil {
m.frames.Stop()
m.frames = nil
}
if m.mon != nil {
m.mon.Stop()
m.mon = nil
}
}
// live reports whether this platform can capture for real.
//
// Both are required. The app is a camera warped by a voice, so without either
// one there is nothing to show — and it says so rather than substituting
// something. A drawing stood in here once; it made the demo look like it worked
// on platforms where it did not.
func (m *mirror) live() bool {
return m.ctx.CameraPreview() != nil && m.ctx.Microphone() != nil
}
func (m *mirror) running() bool { return m.source != nil || m.frames != nil }
// start opens both streams. It must be called from a tap: browsers only honour
// getUserMedia inside a user gesture, so there is no starting this on mount.
func (m *mirror) start() {
if m.starting || m.running() {
return
}
m.SetState(func() { m.starting, m.err = true, "" })
m.ctx.CameraPreview().Start(shell.PreviewOptions{Facing: shell.FacingFront, Width: 640},
func(f shell.Frames, err error) {
m.SetState(func() {
m.starting = false
if err != nil {
m.err = "camera: " + err.Error()
return
}
m.frames = f
})
})
mic := m.ctx.Microphone()
if mic == nil {
return // camera-only platform: the effect runs, it just does not breathe
}
// Ask before listening, exactly as the camera does above.
//
// Skipping this went unnoticed for a long time because the two platforms
// hide it differently. Android refuses outright, which looked like a
// platform quirk and got worked around by granting the permission by hand.
// iOS does something worse: an audio session activated without permission
// still starts, still delivers buffers, and every sample in them is zero —
// so the microphone appears connected and the picture simply never moves.
mic.Authorize(func(p shell.Permission) {
if p != shell.PermissionGranted {
m.SetState(func() { m.err = "microphone: permission denied" })
return
}
mic.Listen(func(mon shell.Monitor, err error) {
m.SetState(func() {
if err != nil {
// A mirror with no microphone is still a mirror; it just
// sits still. Losing the camera is fatal, the mic is not.
m.err = "microphone: " + err.Error()
return
}
m.mon = mon
})
})
})
}
// Tick pulls a frame and a spectrum, warps one into the other, and asks for a
// repaint. Everything here is polled rather than pushed: nothing is captured
// for a frame the app was never going to draw.
func (m *mirror) Tick(dt float64) bool {
if dt > 0.1 {
dt = 0.1
}
m.t += float32(dt)
if dt > 0 {
m.fps += (float32(1/dt) - m.fps) * 0.05
}
m.autostart()
if !m.running() {
return true
}
m.readAudio()
src := m.readFrame()
if src == nil {
m.ctx.Invalidate()
return true
}
dst := m.buffer(src.Bounds())
start := time.Now()
Warp(dst, src, Params{
Level: m.smooth, Bands: m.bands, T: m.t,
Amount: m.amount, Mirror: m.mirrored,
})
m.lastWarp = time.Since(start)
m.show = dst
// The painter caches a copy of an image's pixels keyed by the image value,
// so a buffer it has already drawn stays frozen at whatever it held then.
// Rotating two buffers does not avoid that — both end up cached, and the
// preview cycles two stale frames forever, which is exactly how this
// looked: the first frames arrived and then the picture stopped.
m.ctx.Painter().ImageChanged(dst)
m.ctx.Invalidate()
return true
}
func (m *mirror) readAudio() {
switch {
case m.source != nil:
m.level = m.source.Level()
m.source.Bands(m.bands)
case m.mon != nil:
m.level = m.mon.Level()
m.mon.Bands(m.bands)
default:
m.level = 0
clear(m.bands)
}
// Attack fast, release slow: the image should jump on a syllable and settle
// afterwards, not chatter at the frame rate.
k := float32(0.35)
if m.level < m.smooth {
k = 0.10
}
m.smooth += (m.level - m.smooth) * k
}
func (m *mirror) readFrame() *image.RGBA {
if m.source != nil {
return m.source.Frame()
}
return m.frames.Frame()
}
// buffer returns the next output image, reallocating only when the camera
// changes frame size.
func (m *mirror) buffer(r image.Rectangle) *image.RGBA {
m.cur = (m.cur + 1) % len(m.out)
if m.out[m.cur] == nil || m.out[m.cur].Bounds() != r {
m.out[m.cur] = image.NewRGBA(r)
}
return m.out[m.cur]
}
// --- Build -------------------------------------------------------------------
func (m *mirror) Build(ctx widget.Ctx) widget.Widget {
th := theme.Dark() // a mirror is a lit rectangle in a dark room
var body widget.Widget
if m.running() {
body = widget.Canvas{Clip: true, Draw: m.draw}
} else {
body = m.idle(th)
}
return widget.Provide[theme.Theme]{Value: th, Child: widget.Fill{Color: bg,
Child: widget.Flex{
Axis: layout.Vertical,
CrossAlign: layout.CrossStretch,
Children: []widget.Widget{
widget.Expand(body),
m.controls(th),
},
}}}
}
func (m *mirror) idle(th theme.Theme) widget.Widget {
label := "Start the mirror"
if m.starting {
label = "Asking permission…"
}
kids := []widget.Widget{
widget.Text{Value: "Mirror", Font: theme.FontBold, Size: th.Type.Display, Color: th.Text},
widget.Sized{H: 8},
widget.Text{Value: "Your camera, warped by your voice. Nothing leaves the device, " +
"and nothing is recorded — the frames are read, drawn, and dropped.",
Size: th.Type.Body, Color: th.Muted, Wrap: true},
widget.Sized{H: 20},
}
if !m.live() {
// No substitute, and no button for a stream that cannot open. Saying so
// is the honest thing a capability-gated demo does.
//
// The mark above it is a closed lens, not a preview: a ring, a pupil and
// a line through it. It exists because this state is what the gallery
// thumbnail renders — headless, with no devices — and a page of nothing
// but grey text reads as an app that crashed rather than one that
// declined. Drawing a face here instead is what the fallback used to do,
// and it is the reason the fallback was removed: it promised a working
// demo the platform could not deliver. A lens with a line through it
// promises nothing.
kids = append([]widget.Widget{closedLens(th), widget.Sized{H: 22}}, kids...)
kids = append(kids, widget.Text{
Value: "This platform has no camera and microphone available to the app yet, so there is nothing to mirror.",
Size: th.Type.Body, Color: th.Muted, Wrap: true,
})
return widget.Center(widget.Sized{W: 420, Child: widget.Padding{All: 24,
Child: widget.Flex{Axis: layout.Vertical, CrossAlign: layout.CrossStart, Children: kids}}})
}
kids = append(kids, theme.Button{Label: label, Primary: true, OnTap: m.start})
if m.err != "" {
kids = append(kids, widget.Sized{H: 14},
widget.Text{Value: m.err, Size: th.Type.Label, Color: th.Danger, Wrap: true})
}
return widget.Center(widget.Sized{W: 420, Child: widget.Padding{All: 24,
Child: widget.Flex{Axis: layout.Vertical, CrossAlign: layout.CrossStart, Children: kids}}})
}
// closedLens draws the shut-camera mark shown when the app cannot run: a ring,
// a pupil, and a line across. Circles come from a rounded rect whose radius is
// half its side, which is the whole trick — there is no circle primitive and
// this needs no path.
func closedLens(th theme.Theme) widget.Widget {
const box = 64
return widget.Sized{W: box, H: box, Child: widget.Canvas{Draw: func(c paint.Canvas, size geom.Size) {
d := min(size.W, size.H)
if d <= 0 {
return
}
ring := th.Muted.WithAlpha(0.55)
full := geom.RectXYWH((size.W-d)/2, (size.H-d)/2, d, d)
c.StrokeRRect(full, d/2, d*0.055, ring)
pupil := d * 0.30
c.FillRRect(geom.RectXYWH(full.Min.X+(d-pupil)/2, full.Min.Y+(d-pupil)/2, pupil, pupil),
pupil/2, th.Muted.WithAlpha(0.28))
// The line runs corner to corner across the ring, inset so its round
// caps sit on the rim rather than poking past it.
in := d * 0.17
c.Line(geom.Pt{X: full.Min.X + in, Y: full.Max.Y - in},
geom.Pt{X: full.Max.X - in, Y: full.Min.Y + in}, d*0.055, ring)
}}}
}
func (m *mirror) controls(th theme.Theme) widget.Widget {
if !m.running() {
return widget.Sized{H: 0}
}
// The rate is a running average, so it means nothing for the first second —
// and nothing at all under the headless thumbnail renderer, which doesn't
// step in real time. Report it once there is something to report.
status := ""
if m.fps >= 1 {
status = fmt.Sprintf("%.0f fps · warp %.1f ms", m.fps, float64(m.lastWarp.Microseconds())/1000)
}
row := widget.Row(
widget.Sized{W: 200, Child: widget.Flex{
Axis: layout.Vertical,
CrossAlign: layout.CrossStretch,
Children: []widget.Widget{
widget.Text{Value: "Effect", Size: th.Type.Caption, Color: th.Muted},
widget.Sized{H: 4},
theme.Slider{Value: m.amount, Label: "Effect amount",
OnChange: func(v float32) { m.SetState(func() { m.amount = v }) }},
},
}},
widget.Sized{W: 18},
theme.Checkbox{Checked: m.mirrored, Label: "Mirror",
OnChange: func(v bool) { m.SetState(func() { m.mirrored = v }) }},
widget.Expand(widget.Align{X: 1, Y: 0.5,
Child: widget.Text{Value: status, Font: "mono", Size: th.Type.Caption, Color: th.Muted}}),
widget.Sized{W: 14},
theme.Button{Label: "Stop", OnTap: func() { m.SetState(m.stop) }},
)
row.CrossAlign = layout.CrossCenter
return widget.Padding{All: 14, Child: row}
}
// draw paints the warped frame to fill the surface, then the spectrum over it.
func (m *mirror) draw(c paint.Canvas, sz geom.Size) {
c.Clear(bg)
if m.show == nil {
c.TextIn("", "waiting for the camera…", geom.Pt{X: 24, Y: sz.H / 2}, 15, dim)
return
}
b := m.show.Bounds()
// Cover the surface, preserving aspect: a mirror with letterbox bars looks
// like a video player, not a mirror.
scale := sz.W / float32(b.Dx())
if s := sz.H / float32(b.Dy()); s > scale {
scale = s
}
w, h := float32(b.Dx())*scale, float32(b.Dy())*scale
c.Image(m.show, geom.RectXYWH((sz.W-w)/2, (sz.H-h)/2, w, h))
m.drawSpectrum(c, sz)
}
func (m *mirror) drawSpectrum(c paint.Canvas, sz geom.Size) {
const pad = 18
maxH := sz.H * 0.14
bw := (sz.W - pad*2) / float32(len(m.bands))
for i, v := range m.bands {
bh := clamp01(v) * maxH
if bh < 2 {
bh = 2
}
x := pad + float32(i)*bw
c.FillRRect(geom.RectXYWH(x, sz.H-pad-bh, bw*0.72, bh), bw*0.3,
barCol.WithAlpha(0.30+0.55*clamp01(v)))
}
}
// Root is the app's widget tree. It lives here rather than in main so that the
// desktop entry point and the generated mobile bind surface build the same one.
func Root() widget.Widget { return App{} }
// Config is the app's window and font configuration, shared by the desktop
// command and the mobile bind surface so the two cannot drift.
func Config() app.Config {
return app.Config{
Title: "Mirror",
AppID: "com.gophics.mirror",
Size: geom.Size{W: 960, H: 720},
Background: bg,
BackgroundDark: bg,
Font: goregular.TTF,
FontFamilies: map[string][]byte{theme.FontBold: gobold.TTF, "mono": gomono.TTF},
}
}