1######################################################################## 2# This GNU makefile drives the build of the sqlite3 WASM 3# components. It is not part of the canonical build process. 4######################################################################## 5SHELL := $(shell which bash 2>/dev/null) 6MAKEFILE := $(lastword $(MAKEFILE_LIST)) 7all: 8release: all 9 10# Emscripten SDK home dir and related binaries... 11EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/src/emsdk $(HOME)/emsdk)) 12emcc.bin ?= $(word 1,$(wildcard $(shell which emcc) $(EMSDK_HOME)/upstream/emscripten/emcc)) 13ifeq (,$(emcc.bin)) 14 $(error Cannot find emcc.) 15endif 16 17wasm-strip ?= $(shell which wasm-strip 2>/dev/null) 18ifeq (,$(filter clean,$(MAKECMDGOALS))) 19ifeq (,$(wasm-strip)) 20 $(info WARNING: *******************************************************************) 21 $(info WARNING: builds using -O2/-O3/-Os/-Oz will minify WASM-exported names,) 22 $(info WARNING: breaking _All The Things_. The workaround for that is to build) 23 $(info WARNING: with -g3 (which explodes the file size) and then strip the debug) 24 $(info WARNING: info after compilation, using wasm-strip, to shrink the wasm file.) 25 $(info WARNING: wasm-strip was not found in the PATH so we cannot strip those.) 26 $(info WARNING: If this build uses any optimization level higher than -O1 then) 27 $(info WARNING: the ***resulting WASM binary WILL NOT BE USABLE***.) 28 $(info WARNING: wasm-strip is part of the wabt package:) 29 $(info WARNING: https://github.com/WebAssembly/wabt) 30 $(info WARNING: on Ubuntu-like systems it can be installed with:) 31 $(info WARNING: sudo apt install wabt) 32 $(info WARNING: *******************************************************************) 33endif 34endif # 'make clean' check 35 36ifeq (,$(wasm-strip)) 37 maybe-wasm-strip = echo "not wasm-stripping" 38else 39 maybe-wasm-strip = $(wasm-strip) 40endif 41 42dir.top := ../.. 43# Reminder: some Emscripten flags require absolute paths 44dir.wasm := $(patsubst %/,%,$(dir $(abspath $(MAKEFILE)))) 45dir.api := api 46dir.jacc := jaccwabyt 47dir.common := common 48dir.fiddle := fiddle 49CLEAN_FILES := *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ 50emcc_enable_bigint ?= 1 51sqlite3.c := $(dir.top)/sqlite3.c 52SQLITE_OPT = \ 53 -DSQLITE_ENABLE_FTS4 \ 54 -DSQLITE_ENABLE_RTREE \ 55 -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ 56 -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ 57 -DSQLITE_ENABLE_STMTVTAB \ 58 -DSQLITE_ENABLE_DBPAGE_VTAB \ 59 -DSQLITE_ENABLE_DBSTAT_VTAB \ 60 -DSQLITE_ENABLE_BYTECODE_VTAB \ 61 -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ 62 -DSQLITE_OMIT_LOAD_EXTENSION \ 63 -DSQLITE_OMIT_DEPRECATED \ 64 -DSQLITE_OMIT_UTF16 \ 65 -DSQLITE_OMIT_SHARED_CACHE \ 66 -DSQLITE_THREADSAFE=0 \ 67 -DSQLITE_TEMP_STORE=3 \ 68 -DSQLITE_OS_KV_OPTIONAL=1 \ 69 '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ 70 -DSQLITE_USE_URI=1 71#SQLITE_OPT += -DSQLITE_ENABLE_MEMSYS5 72# ^^^ MEMSYS5 is hypothetically useful for non-Emscripten builds but 73# requires adding more infrastructure and fixing one spot in the 74# sqlite3 internals which calls malloc() early on. 75 76# SQLITE_OMIT_LOAD_EXTENSION: if this is true, sqlite3_vfs::xDlOpen 77# and friends may be NULL. 78 79ifneq (,$(filter release,$(MAKECMDGOALS))) 80emcc_opt ?= -Oz -g3 -flto 81else 82emcc_opt ?= -O0 -g 83# ^^^^ build times for -O levels higher than 0 are painful at 84# dev-time. 85endif 86# ^^^ -flto improves runtime speed at -O0 considerably but doubles 87# build time. 88# 89# ^^^^ -O3, -Oz, -Os minify symbol names and there appears to be no 90# way around that except to use -g3, but -g3 causes the binary file 91# size to absolutely explode (approx. 5x larger). This minification 92# utterly breaks the resulting module, making it unsable except as 93# self-contained/self-referential-only code, as ALL of the exported 94# symbols get minified names. 95# 96# However, we have an option for using -Oz or -Os: 97# 98# Build with (-Os -g3) or (-Oz -g3) then use wasm-strip, from the wabt 99# tools package (https://github.com/WebAssembly/wabt), to strip the 100# debugging symbols. That results in a small build with unmangled 101# symbol names. -Oz gives ever-so-slightly better compression than 102# -Os: not quite 1% in some completely unscientific tests. Runtime 103# speed for the unit tests is all over the place either way so it's 104# difficult to say whether -Os gives any speed benefit over -Oz. 105######################################################################## 106 107 108$(sqlite3.c): 109 $(MAKE) -C $(dir.top) sqlite3.c 110 111clean: 112 -rm -f $(CLEAN_FILES) 113 114ifeq (release,$(filter release,$(MAKECMDGOALS))) 115 ifeq (,$(wasm-strip)) 116 $(error Cannot make release-quality binary because wasm-strip is not available. \ 117 See notes in the warning above) 118 endif 119else 120 $(info Development build. Use '$(MAKE) release' for a smaller release build.) 121endif 122 123EXPORTED_FUNCTIONS.api.in := $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api \ 124 $(dir.jacc)/jaccwabyt_test.exports 125 126EXPORTED_FUNCTIONS.api: $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) 127 cat $(EXPORTED_FUNCTIONS.api.in) > $@ 128CLEAN_FILES += EXPORTED_FUNCTIONS.api 129 130sqlite3-api.jses := $(dir.api)/sqlite3-api-prologue.js 131sqlite3-api.jses += $(dir.common)/whwasmutil.js 132sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js 133sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js 134sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js 135sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js 136sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js 137sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js 138 139sqlite3-api.js := sqlite3-api.js 140CLEAN_FILES += $(sqlite3-api.js) 141$(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) 142 @echo "Making $@..." 143 @for i in $(sqlite3-api.jses); do \ 144 echo "/* BEGIN FILE: $$i */"; \ 145 cat $$i; \ 146 echo "/* END FILE: $$i */"; \ 147 done > $@ 148 149post-js.js := post-js.js 150CLEAN_FILES += $(post-js.js) 151post-jses := \ 152 $(dir.api)/post-js-header.js \ 153 $(sqlite3-api.js) \ 154 $(dir.api)/post-js-footer.js 155 156$(post-js.js): $(post-jses) $(MAKEFILE) 157 @echo "Making $@..." 158 @for i in $(post-jses); do \ 159 echo "/* BEGIN FILE: $$i */"; \ 160 cat $$i; \ 161 echo "/* END FILE: $$i */"; \ 162 done > $@ 163 164 165######################################################################## 166# emcc flags for .c/.o/.wasm/.js. 167emcc.flags = 168#emcc.flags += -v # _very_ loud but also informative about what it's doing 169# -g is needed to keep -O2 and higher from creating broken JS via 170# minification. 171 172######################################################################## 173# emcc flags for .c/.o. 174emcc.cflags := 175emcc.cflags += -std=c99 -fPIC 176# -------------^^^^^^^^ we currently need c99 for WASM-specific sqlite3 APIs. 177emcc.cflags += -I. -I$(dir.top) 178 179######################################################################## 180# emcc flags specific to building the final .js/.wasm file... 181emcc.jsflags := -fPIC 182emcc.jsflags += --minify 0 183emcc.jsflags += --no-entry 184emcc.jsflags += -sMODULARIZE 185emcc.jsflags += -sSTRICT_JS 186emcc.jsflags += -sDYNAMIC_EXECUTION=0 187emcc.jsflags += -sNO_POLYFILL 188emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api 189emcc.exportedRuntimeMethods := \ 190 -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack 191 # FS ==> stdio/POSIX I/O proxies 192 # wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY 193 # allocateUTF8OnStack => for kvvfs internals 194emcc.jsflags += $(emcc.exportedRuntimeMethods) 195emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 196emcc.jsflags += -sIMPORTED_MEMORY 197emcc.environment := -sENVIRONMENT=web 198emcc.jsflags += -sALLOW_MEMORY_GROWTH 199# emcc: warning: USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code 200# slowly, see https://github.com/WebAssembly/design/issues/1271 201# [-Wpthreads-mem-growth] 202emcc.jsflags += -sINITIAL_MEMORY=13107200 203#emcc.jsflags += -sINITIAL_MEMORY=64225280 204# ^^^^ 64MB is not enough for WASMFS/OPFS test runs using batch-runner.js 205emcc.jsflags += $(emcc.environment) 206#emcc.jsflags += -sTOTAL_STACK=4194304 207emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule 208emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. 209emcc.jsflags += --post-js=$(post-js.js) 210#emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() 211#emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS 212#emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API 213#emcc.jsflags += -sABORTING_MALLOC 214emcc.jsflags += -sALLOW_TABLE_GROWTH 215emcc.jsflags += -Wno-limited-postlink-optimizations 216# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. 217#emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why 218# https://lld.llvm.org/WebAssembly.html 219emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 220emcc.jsflags += -sLLD_REPORT_UNDEFINED 221#emcc.jsflags += --allow-undefined 222emcc.jsflags += --import-undefined 223#emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic 224#emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined 225#emcc.jsflags += --unresolved-symbols=ignore-all 226emcc.jsflags += -sWASM_BIGINT=$(emcc_enable_bigint) 227# ^^^^ MEMORY64=1 fails to load, erroring with: 228# invalid memory limits flags 0x5 229# (enable via --experimental-wasm-memory64) 230# 231# ^^^^ MEMORY64=2 builds and loads but dies when we do things like: 232# 233# new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) 234# 235# because ptr is now a BigInt, so is invalid for passing to arguments 236# which have strict must-be-a-Number requirements. 237######################################################################## 238 239 240######################################################################## 241# -sSINGLE_FILE: 242# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js#L1704 243# -sSINGLE_FILE=1 would be really nice but we have to build with -g 244# for -O2 and higher to work (else minification breaks the code) and 245# cannot wasm-strip the binary before it gets encoded into the JS 246# file. The result is that the generated JS file is, because of the -g 247# debugging info, _huge_. 248######################################################################## 249 250######################################################################## 251# Maintenance reminder: the output .js and .wasm files of emcc must be 252# in _this_ dir, rather than a subdir, or else parts of the generated 253# code get confused and cannot load property (namely, the 254# sqlite3.worker.js generated in conjunction with -sWASMFS). 255sqlite3.js := sqlite3.js 256sqlite3.wasm := sqlite3.wasm 257sqlite3-wasm.o := $(dir.api)/sqlite3-wasm.o 258$(sqlite3-wasm.o): emcc.cflags += $(SQLITE_OPT) 259$(sqlite3-wasm.o): $(dir.top)/sqlite3.c 260sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c 261jaccwabyt_test.c := $(dir.jacc)/jaccwabyt_test.c 262# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test apps, 263# so we don't really want to include it in release builds. However, we 264# want to test the release builds with those apps, so we cannot simply 265# elide that file in release builds. That component is critical to the 266# VFS bindings so needs to be tested along with the core APIs. 267define WASM_C_COMPILE 268$(1).o := $$(subst .c,.o,$(1)) 269sqlite3.wasm.obj += $$($(1).o) 270$$($(1).o): $$(MAKEFILE) $(1) 271 $$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@ 272CLEAN_FILES += $$($(1).o) 273endef 274$(foreach c,$(sqlite3-wasm.c) $(jaccwabyt_test.c),$(eval $(call WASM_C_COMPILE,$(c)))) 275$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \ 276 EXPORTED_FUNCTIONS.api \ 277 $(post-js.js) 278 @echo "Building $@ ..." 279 $(emcc.bin) -o $(sqlite3.js) $(emcc_opt) $(emcc.flags) $(emcc.jsflags) $(sqlite3.wasm.obj) 280 chmod -x $(sqlite3.wasm) 281 $(maybe-wasm-strip) $(sqlite3.wasm) 282 @ls -la $@ $(sqlite3.wasm) 283 284CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm) 285all: $(sqlite3.js) 286wasm: $(sqlite3.js) 287# End main Emscripten-based module build 288######################################################################## 289 290######################################################################## 291# batch-runner.js... 292dir.sql := sql 293speedtest1 := ../../speedtest1 294speedtest1.c := ../../test/speedtest1.c 295speedtest1.sql := $(dir.sql)/speedtest1.sql 296speedtest1.cliflags := --size 50 --big-transactions 297$(speedtest1): 298 $(MAKE) -C ../.. speedtest1 299$(speedtest1.sql): $(speedtest1) $(MAKEFILE) 300 $(speedtest1) $(speedtest1.cliflags) --script $@ 301batch-runner.list: $(MAKEFILE) $(speedtest1.sql) $(dir.sql)/000-mandelbrot.sql 302 bash split-speedtest1-script.sh $(dir.sql)/speedtest1.sql 303 ls -1 $(dir.sql)/*.sql | grep -v speedtest1.sql | sort > $@ 304clean-batch: 305 rm -f batch-runner.list $(dir.sql)/speedtest1*.sql 306# ^^^ we don't do this along with 'clean' because we clean/rebuild on 307# a regular basis with different -Ox flags and rebuilding the batch 308# pieces each time is an unnecessary time sink. 309batch: batch-runner.list 310all: batch 311# end batch-runner.js 312######################################################################## 313# speedtest1.js... 314# speedtest1-common.eflags = emcc flags used by multiple builds of speedtest1 315# speedtest1.eflags = emcc flags used by main build of speedtest1 316speedtest1-common.eflags := -g $(emcc_opt) 317speedtest1.eflags := 318speedtest1.eflags += -sENVIRONMENT=web 319speedtest1-common.eflags += -sINVOKE_RUN=0 320speedtest1-common.eflags += --no-entry 321#speedtest1-common.eflags += -flto 322speedtest1-common.eflags += -sABORTING_MALLOC 323speedtest1-common.eflags += -sINITIAL_MEMORY=128450560 324speedtest1-common.eflags += -sSTRICT_JS 325speedtest1-common.eflags += -sMODULARIZE 326speedtest1-common.eflags += -Wno-limited-postlink-optimizations 327speedtest1-common.eflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.speedtest1 328speedtest1-common.eflags += $(emcc.exportedRuntimeMethods) 329speedtest1-common.eflags += -sALLOW_TABLE_GROWTH 330speedtest1-common.eflags += -sDYNAMIC_EXECUTION=0 331speedtest1-common.eflags += --minify 0 332speedtest1-common.eflags += -sEXPORT_NAME=sqlite3Speedtest1InitModule 333speedtest1-common.eflags += --post-js=$(post-js.js) 334speedtest1-common.eflags += -sWASM_BIGINT=$(emcc_enable_bigint) 335speedtest1.exit-runtime0 := -sEXIT_RUNTIME=0 336speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 337# Re -sEXIT_RUNTIME=1 vs 0: if it's 1 and speedtest1 crashes, we get 338# this error from emscripten: 339# 340# > native function `free` called after runtime exit (use 341# NO_EXIT_RUNTIME to keep it alive after main() exits)) 342# 343# If it's 0 and it crashes, we get: 344# 345# > stdio streams had content in them that was not flushed. you should 346# set EXIT_RUNTIME to 1 (see the FAQ), or make sure to emit a newline 347# when you printf etc. 348# 349# and pending output is not flushed because it didn't end with a 350# newline (by design). The lesser of the two evils seems to be 351# -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app 352# which runs speedtest1 multiple times. 353 354EXPORTED_FUNCTIONS.speedtest1: EXPORTED_FUNCTIONS.api 355 { echo _wasm_main; cat EXPORTED_FUNCTIONS.api; } > $@ 356CLEAN_FILES += EXPORTED_FUNCTIONS.speedtest1 357speedtest1.js := speedtest1.js 358speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js)) 359speedtest1.cflags := \ 360 -I. -I.. -I$(dir.top) \ 361 -DSQLITE_SPEEDTEST1_WASM 362speedtest1.cs := $(speedtest1.c) $(sqlite3-wasm.c) $(jaccwabyt_test.c) 363$(speedtest1.js): emcc.cflags+= 364# speedtest1 notes re. sqlite3-wasm.o vs sqlite3-wasm.c: building against 365# the latter (predictably) results in a slightly faster binary, but we're 366# close enough to the target speed requirements that the 500ms makes a 367# difference. 368$(speedtest1.js): $(MAKEFILE) $(speedtest1.cs) $(post-js.js) \ 369 EXPORTED_FUNCTIONS.speedtest1 370 @echo "Building $@ ..." 371 $(emcc.bin) \ 372 $(speedtest1.eflags) $(speedtest1-common.eflags) $(speedtest1.cflags) \ 373 $(SQLITE_OPT) \ 374 $(speedtest1.exit-runtime0) \ 375 -o $@ $(speedtest1.cs) -lm 376 $(maybe-wasm-strip) $(speedtest1.wasm) 377 ls -la $@ $(speedtest1.wasm) 378 379speedtest1: $(speedtest1.js) 380all: speedtest1 381CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) 382# end speedtest1.js 383######################################################################## 384 385######################################################################## 386# fiddle_remote is the remote destination for the fiddle app. It 387# must be a [user@]HOST:/path for rsync. 388# Note that the target "should probably" contain a symlink of 389# index.html -> fiddle.html. 390fiddle_remote ?= 391ifeq (,$(fiddle_remote)) 392ifneq (,$(wildcard /home/stephan)) 393 fiddle_remote = wh:www/wh/sqlite3/. 394else ifneq (,$(wildcard /home/drh)) 395 #fiddle_remote = if appropriate, add that user@host:/path here 396endif 397endif 398$(fiddle_files): default 399push-fiddle: $(fiddle_files) 400 @if [ x = "x$(fiddle_remote)" ]; then \ 401 echo "fiddle_remote must be a [user@]HOST:/path for rsync"; \ 402 exit 1; \ 403 fi 404 rsync -va fiddle/ $(fiddle_remote) 405# end fiddle remote push 406######################################################################## 407 408######################################################################## 409# Convenience rules to rebuild with various -Ox levels. Much 410# experimentation shows -O2 to be the clear winner in terms of speed. 411# Note that build times with anything higher than -O0 are somewhat 412# painful. 413 414.PHONY: o0 o1 o2 o3 os oz 415o-xtra := 416#o-xtra += -flto 417o0: 418 $(MAKE) clean; $(MAKE) -e "emcc_opt=-O0 $(o-xtra)" fiddle_opt=-O0 419o1: 420 $(MAKE) clean; $(MAKE) -e "emcc_opt=-O1 $(o-xtra)" fiddle_opt=-O1 421o2: 422 $(MAKE) clean; $(MAKE) -e "emcc_opt=-O2 $(o-xtra)" fiddle_opt=-O2 423o3: 424 $(MAKE) clean; $(MAKE) -e "emcc_opt=-O3 $(o-xtra)" fiddle_opt=-O3 425os: 426 $(MAKE) clean; $(MAKE) -e "emcc_opt=-Os $(o-xtra)" fiddle_opt=-Os 427oz: 428 $(MAKE) clean; $(MAKE) -e "emcc_opt=-Oz $(o-xtra)" fiddle_opt=-Oz 429 430######################################################################## 431# Sub-makes... 432 433include fiddle.make 434 435######################################################################## 436# Some platforms do not support the WASMFS build. Raspberry Pi OS is one 437# of them. As such platforms are discovered, add their (uname -m) name 438# to PLATFORMS_WITH_NO_WASMFS to exclude the wasmfs build parts. 439PLATFORMS_WITH_NO_WASMFS := aarch64 # add any others here 440THIS_ARCH := $(shell /usr/bin/uname -m) 441ifneq (,$(filter $(THIS_ARCH),$(PLATFORMS_WITH_NO_WASMFS))) 442$(info This platform does not support the WASMFS build.) 443else 444include wasmfs.make 445endif 446 447######################################################################## 448# Push files to public wasm-testing.sqlite.org server 449wasm-testing.include = *.wasm *.js *.html \ 450 batch-runner.list sql common fiddle jaccwabyt 451wasm-testing.exclude = sql/speedtest1.sql 452wasm-testing.dir = /jail/sites/wasm-testing 453wasm-testing.dest ?= wasm-testing:$(wasm-testing.dir) 454# ---------------------^^^^^^^^^^^^ ssh alias 455push: 456 rsync -z -e ssh --ignore-times --chown=stephan:www-data --group -r \ 457 $(patsubst %,--exclude=%,$(wasm-testing.exclude)) \ 458 $(wasm-testing.include) $(wasm-testing.dest) 459 ssh wasm-testing 'cd $(wasm-testing.dir) && bash .gzip' || \ 460 echo "SSH failed: it's likely that stale content will be served via old gzip files." 461 462