Merge 'Add embedded library support to Go adapter' from Jonathan Ness

This change enables the Go adapter to embed platform-specific libraries
and extract them at runtime, eliminating the need for users to set
LD_LIBRARY_PATH or other environment variables.
- Add embedded.go with core library extraction functionality
- Update limbo_unix.go and limbo_windows.go to use embedded libraries
- Add build_lib.sh script to generate platform-specific libraries
- Update README.md with documentation for the new feature
- Add .gitignore to prevent committing binary files
- Add test coverage for Vector operations (vector(), vector_extract(),
vector_distance_cos()) and sqlite core features
The implementation maintains backward compatibility with the traditional
library loading mechanism as a fallback. This approach is inspired by
projects like go-embed-python that use a similar technique for native
library distribution.
https://github.com/tursodatabase/limbo/issues/506

Reviewed-by: Preston Thorpe (@PThorpe92)

Closes #1434

bindings/go/.gitignore

@@ -0,0 +1,4 @@
+# Ignore generated libraries directory
+/libs/*
+# But keep the directory structure
+!/libs/.gitkeep

bindings/go/README.md

@@ -1,14 +1,56 @@
 # Limbo driver for Go's `database/sql` library
 
-
 **NOTE:** this is currently __heavily__ W.I.P and is not yet in a usable state.
 
 This driver uses the awesome [purego](https://github.com/ebitengine/purego) library to call C (in this case Rust with C ABI) functions from Go without the use of `CGO`.
 
+## Embedded Library Support
+
+This driver includes an embedded library feature that allows you to distribute a single binary without requiring users to set environment variables. The library for your platform is automatically embedded, extracted at runtime, and loaded dynamically.
+
+### Building from Source
+
+To build with embedded library support, follow these steps:
+
+```bash
+# Clone the repository
+git clone https://github.com/tursodatabase/limbo
+
+# Navigate to the Go bindings directory
+cd limbo/bindings/go
+
+# Build the library (defaults to release build)
+./build_lib.sh
+
+# Alternatively, for faster builds during development:
+./build_lib.sh debug
+```
+
+### Build Options:
+
+* Release Build (default): ./build_lib.sh or ./build_lib.sh release
+
+    - Optimized for performance and smaller binary size
+    - Takes longer to compile and requires more system resources
+    - Recommended for production use
+
+* Debug Build: ./build_lib.sh debug
+
+    - Faster compilation times with less resource usage
+    - Larger binary size and slower runtime performance
+    - Recommended during development or if release build fails
+
+If the embedded library cannot be found or extracted, the driver will fall back to the traditional method of finding the library in the system paths.
 
 ## To use: (_UNSTABLE_ testing or development purposes only)
 
-### Linux | MacOS
+### Option 1: Using the embedded library (recommended)
+
+Build the driver with the embedded library as described above, then simply import and use. No environment variables needed!
+
+### Option 2: Manual library setup
+
+#### Linux | MacOS
 
 _All commands listed are relative to the bindings/go directory in the limbo repository_
 
@@ -21,7 +63,7 @@ export LD_LIBRARY_PATH="/path/to/limbo/target/debug:$LD_LIBRARY_PATH"
 
 ```
 
-## Windows
+#### Windows
 
 ```
 cargo build --package limbo-go
@@ -69,3 +111,7 @@ func main() {
     }
 }
 ```
+
+## Implementation Notes
+
+The embedded library feature was inspired by projects like [go-embed-python](https://github.com/kluctl/go-embed-python), which uses a similar approach for embedding and distributing native libraries with Go applications.

bindings/go/build_lib.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+# bindings/go/build_lib.sh
+
+set -e
+
+# Accept build type as parameter, default to release
+BUILD_TYPE=${1:-release}
+
+echo "Building Limbo Go library for current platform (build type: $BUILD_TYPE)..."
+
+# Determine platform-specific details
+case "$(uname -s)" in
+    Darwin*)
+        OUTPUT_NAME="lib_limbo_go.dylib"
+        # Map x86_64 to amd64 for Go compatibility
+        ARCH=$(uname -m)
+        if [ "$ARCH" == "x86_64" ]; then
+            ARCH="amd64"
+        fi
+        PLATFORM="darwin_${ARCH}"
+        ;;
+    Linux*)
+        OUTPUT_NAME="lib_limbo_go.so"
+        # Map x86_64 to amd64 for Go compatibility
+        ARCH=$(uname -m)
+        if [ "$ARCH" == "x86_64" ]; then
+            ARCH="amd64"
+        fi
+        PLATFORM="linux_${ARCH}"
+        ;;
+    MINGW*|MSYS*|CYGWIN*)
+        OUTPUT_NAME="lib_limbo_go.dll"
+        if [ "$(uname -m)" == "x86_64" ]; then
+            PLATFORM="windows_amd64"
+        else
+            PLATFORM="windows_386"
+        fi
+        ;;
+    *)
+        echo "Unsupported platform: $(uname -s)"
+        exit 1
+        ;;
+esac
+
+# Create output directory
+OUTPUT_DIR="libs/${PLATFORM}"
+mkdir -p "$OUTPUT_DIR"
+
+# Set cargo build arguments based on build type
+if [ "$BUILD_TYPE" == "debug" ]; then
+    CARGO_ARGS=""
+    TARGET_DIR="debug"
+    echo "NOTE: Debug builds are faster to compile but less efficient at runtime."
+    echo "      For production use, consider using a release build with: ./build_lib.sh release"
+else
+    CARGO_ARGS="--release"
+    TARGET_DIR="release"
+    echo "NOTE: Release builds may take longer to compile and require more system resources."
+    echo "      If this build fails or takes too long, try a debug build with: ./build_lib.sh debug"
+fi
+
+# Build the library
+echo "Running cargo build ${CARGO_ARGS} --package limbo-go"
+cargo build ${CARGO_ARGS} --package limbo-go
+
+# Copy to the appropriate directory
+echo "Copying $OUTPUT_NAME to $OUTPUT_DIR/"
+cp "../../target/${TARGET_DIR}/$OUTPUT_NAME" "$OUTPUT_DIR/"
+
+echo "Library built successfully for $PLATFORM ($BUILD_TYPE build)"

bindings/go/embedded.go

@@ -0,0 +1,133 @@
+// Go bindings for the Limbo database.
+//
+// This file implements library embedding and extraction at runtime, a pattern
+// also used in several other Go projects that need to distribute native binaries:
+//
+//   - github.com/kluctl/go-embed-python: Embeds a full Python distribution in Go
+//     binaries, extracting to temporary directories at runtime. The approach used here
+//     was directly inspired by its embed_util implementation.
+//
+//   - github.com/kluctl/go-jinja2: Uses the same pattern to embed Jinja2 and related
+//     Python libraries, allowing Go applications to use Jinja2 templates without
+//     external dependencies.
+//
+// This approach has several advantages:
+// - Allows distribution of a single, self-contained binary
+// - Eliminates the need for users to set LD_LIBRARY_PATH or other environment variables
+// - Works cross-platform with the same codebase
+// - Preserves backward compatibility with existing methods
+// - Extracts libraries only once per execution via sync.Once
+//
+// The embedded library is extracted to a user-specific temporary directory and
+// loaded dynamically. If extraction fails, the code falls back to the traditional
+// method of searching system paths.
+package limbo
+
+import (
+	"embed"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+)
+
+//go:embed libs/*
+var embeddedLibs embed.FS
+
+var (
+	extractOnce   sync.Once
+	extractedPath string
+	extractErr    error
+)
+
+// extractEmbeddedLibrary extracts the library for the current platform
+// to a temporary directory and returns the path to the extracted library
+func extractEmbeddedLibrary() (string, error) {
+	extractOnce.Do(func() {
+		// Determine platform-specific details
+		var libName string
+		var platformDir string
+
+		switch runtime.GOOS {
+		case "darwin":
+			libName = "lib_limbo_go.dylib"
+		case "linux":
+			libName = "lib_limbo_go.so"
+		case "windows":
+			libName = "lib_limbo_go.dll"
+		default:
+			extractErr = fmt.Errorf("unsupported operating system: %s", runtime.GOOS)
+			return
+		}
+
+		// Determine architecture suffix
+		var archSuffix string
+		switch runtime.GOARCH {
+		case "amd64":
+			archSuffix = "amd64"
+		case "arm64":
+			archSuffix = "arm64"
+		case "386":
+			archSuffix = "386"
+		default:
+			extractErr = fmt.Errorf("unsupported architecture: %s", runtime.GOARCH)
+			return
+		}
+
+		// Create platform directory string
+		platformDir = fmt.Sprintf("%s_%s", runtime.GOOS, archSuffix)
+
+		// Create a unique temporary directory for the current user
+		tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("limbo-go-%d", os.Getuid()))
+		if err := os.MkdirAll(tempDir, 0755); err != nil {
+			extractErr = fmt.Errorf("failed to create temp directory: %w", err)
+			return
+		}
+
+		// Path to the library within the embedded filesystem
+		libPath := filepath.Join("libs", platformDir, libName)
+
+		// Where the library will be extracted
+		extractedPath = filepath.Join(tempDir, libName)
+
+		// Check if library already exists and is valid
+		if stat, err := os.Stat(extractedPath); err == nil && stat.Size() > 0 {
+			// Library already exists, nothing to do
+			return
+		}
+
+		// Open the embedded library
+		embeddedLib, err := embeddedLibs.Open(libPath)
+		if err != nil {
+			extractErr = fmt.Errorf("failed to open embedded library %s: %w", libPath, err)
+			return
+		}
+		defer embeddedLib.Close()
+
+		// Create the output file
+		outFile, err := os.Create(extractedPath)
+		if err != nil {
+			extractErr = fmt.Errorf("failed to create output file: %w", err)
+			return
+		}
+		defer outFile.Close()
+
+		// Copy the library to the temporary directory
+		if _, err := io.Copy(outFile, embeddedLib); err != nil {
+			extractErr = fmt.Errorf("failed to extract library: %w", err)
+			return
+		}
+
+		// On Unix systems, make the library executable
+		if runtime.GOOS != "windows" {
+			if err := os.Chmod(extractedPath, 0755); err != nil {
+				extractErr = fmt.Errorf("failed to make library executable: %w", err)
+				return
+			}
+		}
+	})
+
+	return extractedPath, extractErr
+}

bindings/go/limbo_test.go

@@ -4,13 +4,16 @@ import (
 	"database/sql"
 	"fmt"
 	"log"
+	"math"
 	"testing"
 
 	_ "github.com/tursodatabase/limbo"
 )
 
-var conn *sql.DB
-var connErr error
+var (
+	conn    *sql.DB
+	connErr error
+)
 
 func TestMain(m *testing.M) {
 	conn, connErr = sql.Open("sqlite3", ":memory:")
@@ -59,7 +62,7 @@ func TestQuery(t *testing.T) {
 			t.Errorf("Expected column %d to be %s, got %s", i, expectedCols[i], col)
 		}
 	}
-	var i = 1
+	i := 1
 	for rows.Next() {
 		var a int
 		var b string
@@ -78,7 +81,6 @@ func TestQuery(t *testing.T) {
 	if err = rows.Err(); err != nil {
 		t.Fatalf("Row iteration error: %v", err)
 	}
-
 }
 
 func TestFunctions(t *testing.T) {
@@ -280,6 +282,257 @@ func TestDriverRowsErrorMessages(t *testing.T) {
 	t.Log("Rows error behavior test passed")
 }
 
+func TestVectorOperations(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("Error opening connection: %v", err)
+	}
+	defer db.Close()
+
+	// Test creating table with vector columns
+	_, err = db.Exec(`CREATE TABLE vector_test (id INTEGER PRIMARY KEY, embedding F32_BLOB(64))`)
+	if err != nil {
+		t.Fatalf("Error creating vector table: %v", err)
+	}
+
+	// Test vector insertion
+	_, err = db.Exec(`INSERT INTO vector_test VALUES (1, vector('[0.1, 0.2, 0.3, 0.4, 0.5]'))`)
+	if err != nil {
+		t.Fatalf("Error inserting vector: %v", err)
+	}
+
+	// Test vector similarity calculation
+	var similarity float64
+	err = db.QueryRow(`SELECT vector_distance_cos(embedding, vector('[0.2, 0.3, 0.4, 0.5, 0.6]')) FROM vector_test WHERE id = 1`).Scan(&similarity)
+	if err != nil {
+		t.Fatalf("Error calculating vector similarity: %v", err)
+	}
+	if similarity <= 0 || similarity > 1 {
+		t.Fatalf("Expected similarity between 0 and 1, got %f", similarity)
+	}
+
+	// Test vector extraction
+	var extracted string
+	err = db.QueryRow(`SELECT vector_extract(embedding) FROM vector_test WHERE id = 1`).Scan(&extracted)
+	if err != nil {
+		t.Fatalf("Error extracting vector: %v", err)
+	}
+	fmt.Printf("Extracted vector: %s\n", extracted)
+}
+
+func TestSQLFeatures(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("Error opening connection: %v", err)
+	}
+	defer db.Close()
+
+	// Create test tables
+	_, err = db.Exec(`
+        CREATE TABLE customers (
+            id INTEGER PRIMARY KEY,
+            name TEXT,
+            age INTEGER
+        )`)
+	if err != nil {
+		t.Fatalf("Error creating customers table: %v", err)
+	}
+
+	_, err = db.Exec(`
+        CREATE TABLE orders (
+            id INTEGER PRIMARY KEY,
+            customer_id INTEGER,
+            amount REAL,
+            date TEXT
+        )`)
+	if err != nil {
+		t.Fatalf("Error creating orders table: %v", err)
+	}
+
+	// Insert test data
+	_, err = db.Exec(`
+        INSERT INTO customers VALUES
+            (1, 'Alice', 30),
+            (2, 'Bob', 25),
+            (3, 'Charlie', 40)`)
+	if err != nil {
+		t.Fatalf("Error inserting customers: %v", err)
+	}
+
+	_, err = db.Exec(`
+        INSERT INTO orders VALUES
+            (1, 1, 100.50, '2024-01-01'),
+            (2, 1, 200.75, '2024-02-01'),
+            (3, 2, 50.25, '2024-01-15'),
+            (4, 3, 300.00, '2024-02-10')`)
+	if err != nil {
+		t.Fatalf("Error inserting orders: %v", err)
+	}
+
+	// Test JOIN
+	rows, err := db.Query(`
+        SELECT c.name, o.amount
+        FROM customers c
+        INNER JOIN orders o ON c.id = o.customer_id
+        ORDER BY o.amount DESC`)
+	if err != nil {
+		t.Fatalf("Error executing JOIN: %v", err)
+	}
+	defer rows.Close()
+
+	// Check JOIN results
+	expectedResults := []struct {
+		name   string
+		amount float64
+	}{
+		{"Charlie", 300.00},
+		{"Alice", 200.75},
+		{"Alice", 100.50},
+		{"Bob", 50.25},
+	}
+
+	i := 0
+	for rows.Next() {
+		var name string
+		var amount float64
+		if err := rows.Scan(&name, &amount); err != nil {
+			t.Fatalf("Error scanning JOIN result: %v", err)
+		}
+		if i >= len(expectedResults) {
+			t.Fatalf("Too many rows returned from JOIN")
+		}
+		if name != expectedResults[i].name || amount != expectedResults[i].amount {
+			t.Fatalf("Row %d: expected (%s, %.2f), got (%s, %.2f)",
+				i, expectedResults[i].name, expectedResults[i].amount, name, amount)
+		}
+		i++
+	}
+
+	// Test GROUP BY with aggregation
+	var count int
+	var total float64
+	err = db.QueryRow(`
+        SELECT COUNT(*), SUM(amount)
+        FROM orders
+        WHERE customer_id = 1
+        GROUP BY customer_id`).Scan(&count, &total)
+	if err != nil {
+		t.Fatalf("Error executing GROUP BY: %v", err)
+	}
+	if count != 2 || total != 301.25 {
+		t.Fatalf("GROUP BY gave wrong results: count=%d, total=%.2f", count, total)
+	}
+}
+
+func TestDateTimeFunctions(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("Error opening connection: %v", err)
+	}
+	defer db.Close()
+
+	// Test date()
+	var dateStr string
+	err = db.QueryRow(`SELECT date('now')`).Scan(&dateStr)
+	if err != nil {
+		t.Fatalf("Error with date() function: %v", err)
+	}
+	fmt.Printf("Current date: %s\n", dateStr)
+
+	// Test date arithmetic
+	err = db.QueryRow(`SELECT date('2024-01-01', '+1 month')`).Scan(&dateStr)
+	if err != nil {
+		t.Fatalf("Error with date arithmetic: %v", err)
+	}
+	if dateStr != "2024-02-01" {
+		t.Fatalf("Expected '2024-02-01', got '%s'", dateStr)
+	}
+
+	// Test strftime
+	var formatted string
+	err = db.QueryRow(`SELECT strftime('%Y-%m-%d', '2024-01-01')`).Scan(&formatted)
+	if err != nil {
+		t.Fatalf("Error with strftime function: %v", err)
+	}
+	if formatted != "2024-01-01" {
+		t.Fatalf("Expected '2024-01-01', got '%s'", formatted)
+	}
+}
+
+func TestMathFunctions(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("Error opening connection: %v", err)
+	}
+	defer db.Close()
+
+	// Test basic math functions
+	var result float64
+	err = db.QueryRow(`SELECT abs(-15.5)`).Scan(&result)
+	if err != nil {
+		t.Fatalf("Error with abs function: %v", err)
+	}
+	if result != 15.5 {
+		t.Fatalf("abs(-15.5) should be 15.5, got %f", result)
+	}
+
+	// Test trigonometric functions
+	err = db.QueryRow(`SELECT round(sin(radians(30)), 4)`).Scan(&result)
+	if err != nil {
+		t.Fatalf("Error with sin function: %v", err)
+	}
+	if math.Abs(result-0.5) > 0.0001 {
+		t.Fatalf("sin(30 degrees) should be about 0.5, got %f", result)
+	}
+
+	// Test power functions
+	err = db.QueryRow(`SELECT pow(2, 3)`).Scan(&result)
+	if err != nil {
+		t.Fatalf("Error with pow function: %v", err)
+	}
+	if result != 8 {
+		t.Fatalf("2^3 should be 8, got %f", result)
+	}
+}
+
+func TestJSONFunctions(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("Error opening connection: %v", err)
+	}
+	defer db.Close()
+
+	// Test json function
+	var valid int
+	err = db.QueryRow(`SELECT json_valid('{"name":"John","age":30}')`).Scan(&valid)
+	if err != nil {
+		t.Fatalf("Error with json_valid function: %v", err)
+	}
+	if valid != 1 {
+		t.Fatalf("Expected valid JSON to return 1, got %d", valid)
+	}
+
+	// Test json_extract
+	var name string
+	err = db.QueryRow(`SELECT json_extract('{"name":"John","age":30}', '$.name')`).Scan(&name)
+	if err != nil {
+		t.Fatalf("Error with json_extract function: %v", err)
+	}
+	if name != "John" {
+		t.Fatalf("Expected 'John', got '%s'", name)
+	}
+
+	// Test JSON shorthand
+	var age int
+	err = db.QueryRow(`SELECT '{"name":"John","age":30}' -> '$.age'`).Scan(&age)
+	if err != nil {
+		t.Fatalf("Error with JSON shorthand: %v", err)
+	}
+	if age != 30 {
+		t.Fatalf("Expected 30, got %d", age)
+	}
+}
+
 func slicesAreEq(a, b []byte) bool {
 	if len(a) != len(b) {
 		fmt.Printf("LENGTHS NOT EQUAL: %d != %d\n", len(a), len(b))

bindings/go/limbo_unix.go

@@ -13,6 +13,21 @@ import (
 )
 
 func loadLibrary() (uintptr, error) {
+	// Try to extract embedded library first
+	libPath, err := extractEmbeddedLibrary()
+	if err == nil {
+		// Successfully extracted embedded library, try to load it
+		slib, dlerr := purego.Dlopen(libPath, purego.RTLD_NOW|purego.RTLD_GLOBAL)
+		if dlerr == nil {
+			return slib, nil
+		}
+		// If loading failed, log the error and fall back to system paths
+		fmt.Printf("Warning: Failed to load embedded library: %v\n", dlerr)
+	} else {
+		fmt.Printf("Warning: Failed to extract embedded library: %v\n", err)
+	}
+
+	// Fall back to original behavior for compatibility
 	var libraryName string
 	switch runtime.GOOS {
 	case "darwin":
@@ -23,7 +38,7 @@ func loadLibrary() (uintptr, error) {
 		return 0, fmt.Errorf("GOOS=%s is not supported", runtime.GOOS)
 	}
 
-	libPath := os.Getenv("LD_LIBRARY_PATH")
+	libPath = os.Getenv("LD_LIBRARY_PATH")
 	paths := strings.Split(libPath, ":")
 	cwd, err := os.Getwd()
 	if err != nil {

bindings/go/limbo_windows.go

@@ -12,17 +12,33 @@ import (
 )
 
 func loadLibrary() (uintptr, error) {
-	libName := fmt.Sprintf("%s.dll", libName)
+	// Try to extract embedded library first
+	libPath, err := extractEmbeddedLibrary()
+	if err == nil {
+		// Successfully extracted embedded library, try to load it
+		slib, dlerr := windows.LoadLibrary(libPath)
+		if dlerr == nil {
+			return uintptr(slib), nil
+		}
+		// If loading failed, log the error and fall back to system paths
+		fmt.Printf("Warning: Failed to load embedded library: %v\n", dlerr)
+	} else {
+		fmt.Printf("Warning: Failed to extract embedded library: %v\n", err)
+	}
+
+	// Fall back to original behavior
+	libraryName := fmt.Sprintf("%s.dll", libName)
+
 	pathEnv := os.Getenv("PATH")
 	paths := strings.Split(pathEnv, ";")
-
 	cwd, err := os.Getwd()
 	if err != nil {
 		return 0, err
 	}
 	paths = append(paths, cwd)
+
 	for _, path := range paths {
-		dllPath := filepath.Join(path, libName)
+		dllPath := filepath.Join(path, libraryName)
 		if _, err := os.Stat(dllPath); err == nil {
 			slib, loadErr := windows.LoadLibrary(dllPath)
 			if loadErr != nil {
@@ -32,5 +48,5 @@ func loadLibrary() (uintptr, error) {
 		}
 	}
 
-	return 0, fmt.Errorf("library %s not found in PATH or CWD", libName)
+	return 0, fmt.Errorf("library %s not found in PATH or CWD", libraryName)
 }