Added support for _POSIX_MAPPED_FILES if defined in source (#564)

Getting rid of them fixed GA Ubuntu, but broke GA MacOS. Let's try a
different strategy.

.gitignore

@@ -18,6 +18,9 @@ models/*
 
 /main
 /quantize
+/magic.dat
 
 arm_neon.h
 compile_commands.json
+CMakeFiles/
+CMakeCache.txt

CMakeLists.txt

@@ -107,7 +107,9 @@ endif()
 add_executable(llama
     main.cpp
     utils.cpp
-    utils.h)
+    utils.h
+    mmap.c
+    mmap.h)
 
 add_executable(quantize
     quantize.cpp

Makefile

@@ -185,14 +185,17 @@ default: main quantize
 ggml.o: ggml.c ggml.h
 	$(CC)  $(CFLAGS)   -c ggml.c -o ggml.o
 
+mmap.o: mmap.c mmap.h
+	$(CC) $(CFLAGS) -c mmap.c -o mmap.o
+
 utils.o: utils.cpp utils.h
 	$(CXX) $(CXXFLAGS) -c utils.cpp -o utils.o
 
 clean:
 	rm -f *.o main quantize
 
-main: main.cpp ggml.o utils.o
-	$(CXX) $(CXXFLAGS) main.cpp ggml.o utils.o -o main $(LDFLAGS)
+main: main.cpp ggml.o utils.o mmap.o
+	$(CXX) $(CXXFLAGS) main.cpp ggml.o utils.o mmap.o -o main $(LDFLAGS)
 	./main -h
 
 quantize: quantize.cpp ggml.o utils.o

ggml.c

@@ -1,7 +1,8 @@
+#define _GNU_SOURCE
 #include "ggml.h"
 
 #if defined(_MSC_VER) || defined(__MINGW32__)
-#include <malloc.h> // using malloc.h with MSC/MINGW
+//#include <malloc.h> // using malloc.h with MSC/MINGW
 #elif !defined(__FreeBSD__) && !defined(__NetBSD__)
 #include <alloca.h>
 #endif

main.cpp

@@ -1,21 +1,41 @@
+#if defined(_MSC_VER) || defined(__MINGW32__)
+#define NOMINMAX
+#endif
+
 #include "ggml.h"
 
 #include "utils.h"
+#include "mmap.h"
 
 #include <cassert>
 #include <cmath>
 #include <cstdio>
+#include <cerrno>
 #include <cstring>
 #include <fstream>
 #include <map>
 #include <string>
 #include <vector>
+#include <atomic>
 
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
+#include <fcntl.h>
 #include <signal.h>
 #include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#elif defined (_POSIX_MAPPED_FILES)
+#include <sys/mman.h>
 #endif
 
+#define ROUNDUP(X, K) (((X) + (K)-1) & -(K))
+#define IS2POW(X) (!((X) & ((X)-1)))
+
+#define MAGIC_PATH "magic.dat"
+#define MAGIC_ADDR (char *)0x330000000000
+#define MAGIC_GRAN 65536
+#define MAGIC_ALGN (sizeof(size_t) * 2)
+
 #define ANSI_COLOR_RED     "\x1b[31m"
 #define ANSI_COLOR_GREEN   "\x1b[32m"
 #define ANSI_COLOR_YELLOW  "\x1b[33m"
@@ -83,6 +103,180 @@ struct llama_model {
     std::map<std::string, struct ggml_tensor *> tensors;
 };
 
+struct magic {
+    uint32_t magic;
+    std::atomic<unsigned> lock;
+    int fd;
+    uint64_t commit;
+    uint64_t offset;
+    uint64_t capacity;
+    gpt_vocab *vocab;
+    llama_model *model;
+};
+
+static struct magic *mag;
+
+static inline void spin_lock(std::atomic<unsigned> &lock) {
+    while (lock.exchange(1, std::memory_order_acquire));
+}
+
+static inline void spin_unlock(std::atomic<unsigned> &lock) {
+    lock.store(0, std::memory_order_release);
+}
+
+static void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
+    void *res;
+    res = mmap(addr, length, prot, flags, fd, offset);
+    if (res != MAP_FAILED) return res;
+    perror("mmap");
+    exit(77);
+}
+
+static void magic_commit(void) {
+    mag->commit = ROUNDUP(mag->offset, MAGIC_GRAN);
+    mag->magic = 0xFEEDABEE;
+    if (msync(mag, mag->commit, MS_ASYNC) == -1) {
+        perror("msync");
+        exit(77);
+    }
+}
+
+static void magic_init(void) {
+    int fd;
+    size_t n;
+    int64_t size;
+    if (mag) return;
+    n = ROUNDUP(sizeof(struct magic), MAGIC_GRAN);
+    if ((fd = open(MAGIC_PATH, O_RDWR)) != -1) {
+        if ((size = lseek(fd, 0, SEEK_END)) == -1) {
+            perror("lseek");
+            exit(77);
+        }
+        if (size >= n) {
+            mag = (struct magic *)Mmap(MAGIC_ADDR, n,
+                                       PROT_READ | PROT_WRITE,
+                                       MAP_PRIVATE | MAP_FIXED, fd, 0);
+            if (mag->magic == 0xFEEDABEE) {
+                mag = (struct magic *)Mmap(MAGIC_ADDR, mag->commit,
+                                           PROT_READ | PROT_WRITE,
+                                           MAP_PRIVATE | MAP_FIXED, fd, 0);
+                madvise(MAGIC_ADDR, mag->capacity, MADV_WILLNEED);
+                mag->offset = mag->commit;
+                mag->capacity = mag->commit;
+                mag->fd = -1;
+                return;
+            }
+        }
+        if (ftruncate(fd, 0) == -1) {
+            perror("ftruncate");
+            exit(77);
+        }
+    } else if ((fd = open(MAGIC_PATH, O_RDWR | O_CREAT | O_TRUNC, 0644)) == -1) {
+        perror(MAGIC_PATH);
+        exit(77);
+    }
+    if (ftruncate(fd, n) == -1) {
+        perror("ftruncate");
+        exit(77);
+    }
+    mag = (struct magic *)Mmap(MAGIC_ADDR, n,
+                               PROT_READ | PROT_WRITE,
+                               MAP_SHARED | MAP_FIXED, fd, 0);
+    mag->offset = n;
+    mag->capacity = n;
+    mag->fd = fd;
+}
+
+void *magic_memalign(size_t a, size_t n) {
+    void *p;
+    static int count;
+    size_t i, j, k, m, c2;
+    magic_init();
+    if (a < MAGIC_ALGN) a = MAGIC_ALGN;
+    while (!IS2POW(a)) ++a;
+    m = n ? n : 1;
+    spin_lock(mag->lock);
+    i = mag->offset;
+    i = i + sizeof(size_t);
+    i = ROUNDUP(i, a);
+    j = ROUNDUP(i + m, MAGIC_GRAN);
+    if (j > mag->capacity) {
+        c2 = mag->capacity;
+        if (!c2) {
+            c2 = MAGIC_GRAN;
+        }
+        while (j > c2) {
+            c2 += c2 >> 4;
+            c2 = ROUNDUP(c2, MAGIC_GRAN);
+        }
+        if (!mag->magic) {
+            if (ftruncate(mag->fd, c2) == -1) {
+                perror("ftruncate");
+                spin_unlock(mag->lock);
+                return 0;
+            }
+            p = mmap(MAGIC_ADDR + mag->capacity,
+                     c2 - mag->capacity, PROT_READ | PROT_WRITE,
+                     MAP_SHARED | MAP_FIXED, mag->fd, mag->capacity);
+        } else {
+            p = mmap(MAGIC_ADDR + mag->capacity,
+                     c2 - mag->capacity, PROT_READ | PROT_WRITE,
+                     MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+        }
+        if (p != MAP_FAILED) {
+            mag->capacity = c2;
+        } else {
+            perror("mmap");
+            spin_unlock(mag->lock);
+            return 0;
+        }
+    }
+    mag->offset = i + m;
+    spin_unlock(mag->lock);
+    p = MAGIC_ADDR + i;
+    ((size_t *)p)[-1] = n;
+    return p;
+}
+
+void *magic_malloc(size_t n) {
+    return magic_memalign(MAGIC_ALGN, n);
+}
+
+void *magic_calloc(size_t n, size_t z) {
+    void *p;
+    if ((p = magic_malloc((n *= z)))) {
+        memset(p, 0, n);
+    }
+    return p;
+}
+
+void magic_free(void *p) {
+    // do nothing
+}
+
+void *magic_realloc(void *p, size_t n) {
+    void *q;
+    if (!p) {
+        return magic_malloc(n);
+    }
+    if (!n) {
+        magic_free(p);
+        return 0;
+    }
+    if ((q = magic_malloc(n))) {
+        memcpy(q, p, ((const size_t *)p)[-1]);
+    }
+    return q;
+}
+
+void* operator new(size_t size) {
+    return magic_malloc(size);
+}
+
+void operator delete(void* p) {
+    magic_free(p);
+}
+
 // load the model's weights from a file
 bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab & vocab, int n_ctx) {
     fprintf(stderr, "%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
@@ -227,7 +421,7 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
     {
         struct ggml_init_params params = {
             /*.mem_size   =*/ ctx_size,
-            /*.mem_buffer =*/ NULL,
+            /*.mem_buffer =*/ magic_malloc(ctx_size),
         };
 
         model.ctx = ggml_init(params);
@@ -786,6 +980,8 @@ const char * llama_print_system_info(void) {
 }
 
 int main(int argc, char ** argv) {
+    magic_init();
+
     ggml_time_init();
     const int64_t t_main_start_us = ggml_time_us();
 
@@ -812,19 +1008,24 @@ int main(int argc, char ** argv) {
 
     int64_t t_load_us = 0;
 
-    gpt_vocab vocab;
-    llama_model model;
-
     // load the model
-    {
+    gpt_vocab *vocab;
+    llama_model *model;
+    if (!mag->magic) {
+        vocab = new gpt_vocab;
+        model = new llama_model;
         const int64_t t_start_us = ggml_time_us();
-
-        if (!llama_model_load(params.model, model, vocab, 512)) {  // TODO: set context from user input ??
+        if (!llama_model_load(params.model, *model, *vocab, 512)) {  // TODO: set context from user input ??
             fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
             return 1;
         }
-
         t_load_us = ggml_time_us() - t_start_us;
+        mag->vocab = vocab;
+        mag->model = model;
+        magic_commit();
+    } else {
+        vocab = mag->vocab;
+        model = mag->model;
     }
 
     // print system information
@@ -842,18 +1043,18 @@ int main(int argc, char ** argv) {
     std::vector<float> logits;
 
     // tokenize the prompt
-    std::vector<gpt_vocab::id> embd_inp = ::llama_tokenize(vocab, params.prompt, true);
+    std::vector<gpt_vocab::id> embd_inp = ::llama_tokenize(*vocab, params.prompt, true);
 
-    params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
+    params.n_predict = std::min(params.n_predict, model->hparams.n_ctx - (int) embd_inp.size());
 
     // tokenize the reverse prompt
-    std::vector<gpt_vocab::id> antiprompt_inp = ::llama_tokenize(vocab, params.antiprompt, false);
+    std::vector<gpt_vocab::id> antiprompt_inp = ::llama_tokenize(*vocab, params.antiprompt, false);
 
     fprintf(stderr, "\n");
     fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str());
     fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
     for (int i = 0; i < (int) embd_inp.size(); i++) {
-        fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], vocab.id_to_token.at(embd_inp[i]).c_str());
+        fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], vocab->id_to_token.at(embd_inp[i]).c_str());
     }
     fprintf(stderr, "\n");
     if (params.interactive) {
@@ -871,7 +1072,7 @@ int main(int argc, char ** argv) {
             fprintf(stderr, "%s: reverse prompt: '%s'\n", __func__, params.antiprompt.c_str());
             fprintf(stderr, "%s: number of tokens in reverse prompt = %zu\n", __func__, antiprompt_inp.size());
             for (int i = 0; i < (int) antiprompt_inp.size(); i++) {
-                fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i], vocab.id_to_token.at(antiprompt_inp[i]).c_str());
+                fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i], vocab->id_to_token.at(antiprompt_inp[i]).c_str());
             }
             fprintf(stderr, "\n");
         }
@@ -883,7 +1084,7 @@ int main(int argc, char ** argv) {
 
     // determine the required inference memory per token:
     size_t mem_per_token = 0;
-    llama_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
+    llama_eval(*model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
 
     int last_n_size = params.repeat_last_n;
     std::vector<gpt_vocab::id> last_n_tokens(last_n_size);
@@ -918,7 +1119,7 @@ int main(int argc, char ** argv) {
         if (embd.size() > 0) {
             const int64_t t_start_us = ggml_time_us();
 
-            if (!llama_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
+            if (!llama_eval(*model, params.n_threads, n_past, embd, logits, mem_per_token)) {
                 fprintf(stderr, "Failed to predict\n");
                 return 1;
             }
@@ -936,14 +1137,14 @@ int main(int argc, char ** argv) {
             const float temp  = params.temp;
             const float repeat_penalty = params.repeat_penalty;
 
-            const int n_vocab = model.hparams.n_vocab;
+            const int n_vocab = model->hparams.n_vocab;
 
             gpt_vocab::id id = 0;
 
             {
                 const int64_t t_start_sample_us = ggml_time_us();
 
-                id = llama_sample_top_p_top_k(vocab, logits.data() + (logits.size() - n_vocab), last_n_tokens, repeat_penalty, top_k, top_p, temp, rng);
+                id = llama_sample_top_p_top_k(*vocab, logits.data() + (logits.size() - n_vocab), last_n_tokens, repeat_penalty, top_k, top_p, temp, rng);
 
                 last_n_tokens.erase(last_n_tokens.begin());
                 last_n_tokens.push_back(id);
@@ -980,7 +1181,7 @@ int main(int argc, char ** argv) {
         // display text
         if (!input_noecho) {
             for (auto id : embd) {
-                printf("%s", vocab.id_to_token[id].c_str());
+                printf("%s", vocab->id_to_token[id].c_str());
             }
             fflush(stdout);
         }
@@ -1018,7 +1219,7 @@ int main(int argc, char ** argv) {
                         buf[n_read+1] = 0;
                     }
 
-                    std::vector<gpt_vocab::id> line_inp = ::llama_tokenize(vocab, buf, false);
+                    std::vector<gpt_vocab::id> line_inp = ::llama_tokenize(*vocab, buf, false);
                     embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
 
                     remaining_tokens -= line_inp.size();
@@ -1050,7 +1251,7 @@ int main(int argc, char ** argv) {
         fprintf(stderr, "%s:    total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
     }
 
-    ggml_free(model.ctx);
+    ggml_free(model->ctx);
 
     return 0;
 }

mmap.c

@@ -0,0 +1,570 @@
+// Lightweight Portable mmap() Polyfill
+//
+// 1. Supports POSIX.1
+//
+//    The baseline POSIX standard doesn't specify MAP_ANONYMOUS. This
+//    library makes sure, on the hypothetical UNIX systems that don't
+//    have it, or on the mainstream UNIX platforms where the user has
+//    chosen to define _POSIX_C_SOURCE that cause headers to undefine
+//    it, this implementation will fallback to creating a secure temp
+//    file, for each anonymous mapping.
+//
+// 2. Supports Windows w/ Visual Studio
+//
+//    On Windows Vista and later an API exists that's almost as good as
+//    mmap(). However code that uses this library should conform to the
+//    subset of behaviors Microsoft accommodates.
+//
+// Caveats
+//
+// - You should just assume the page size is 64kb. That's how it is on
+//   Windows and it usually goes faster to assume that elsewhere too.
+//
+// - Not designed to support mprotect() at the moment. In order to
+//   support this, we'd need to consider _open(O_ACCMODE) on Windows
+//   and then have mmap() be more greedy about permissions.
+//
+// - There's limited support for being clever with memory intervals.
+//   For example, you can't punch a hole in a memory map on Windows.
+//   This abstraction does aim to offer more flexibility than WIN32.
+//   There should also be good error reporting for unsupported uses.
+
+#include "mmap.h"
+
+#ifdef NEED_POSIX_MMAP
+#include <stdlib.h>
+
+void *PosixMmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
+    int tfd;
+    void* res;
+    char path[] = "/tmp/llama.dat.XXXXXX";
+    if (~flags & MAP_ANONYMOUS) {
+        res = mmap(addr, length, prot, flags, fd, offset);
+    } else if ((tfd = mkstemp(path)) != -1) {
+        unlink(path);
+        if (!ftruncate(tfd, length)) {
+            res = mmap(addr, length, prot, flags & ~MAP_ANONYMOUS, tfd, 0);
+        } else {
+            res = MAP_FAILED;
+        }
+        close(tfd);
+    } else {
+        res = MAP_FAILED;
+    }
+    return res;
+}
+
+#elif defined(NEED_WIN32_MMAP)
+#include <errno.h>
+#include <stdio.h>
+#include <assert.h>
+#include <inttypes.h>
+
+struct WinMap {        // O(n) no ordering no overlaps
+    HANDLE hand;       // zero means array slots empty
+    HANDLE fand;       // for the original file, or -1
+    uintptr_t addr;    // base address (64 kb aligned)
+    uintptr_t length;  // byte size (>0, rounded 64kb)
+};
+
+struct WinMaps {
+    int n;
+    struct WinMap *p;
+    volatile long lock;
+};
+
+static struct WinMaps g_winmaps;
+
+static inline uintptr_t Min(uintptr_t x, uintptr_t y) {
+    return y > x ? x : y;
+}
+
+static inline uintptr_t Max(uintptr_t x, uintptr_t y) {
+    return y < x ? x : y;
+}
+
+static inline uintptr_t Roundup(uintptr_t x, intptr_t a) {
+    assert(a > 0);
+    assert(!(a & (a - 1)));
+    return (x + (a - 1)) & -a;
+}
+
+static inline void Lock(void) {
+    long x;
+    for (;;) {
+        x = InterlockedExchange(&g_winmaps.lock, 1);
+        if (!x) break;
+        assert(x == 1);
+    }
+}
+
+static inline void Unlock(void) {
+    assert(g_winmaps.lock == 1);
+    g_winmaps.lock = 0;
+}
+
+static int WinStrerror(int err, char *buf, int size) {
+    return FormatMessageA(
+        FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+        NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        buf, size, NULL);
+}
+
+#ifdef NDEBUG
+#define LogError(thing) (void)0
+#else
+static void LogError(const char* file, int line, const char* thing) {
+#define LogError(thing) LogError(__FILE__, __LINE__, thing)
+    fprintf(stderr, "%s:%d: error: %s\n", file, line, thing);
+}
+#endif
+
+#ifdef NDEBUG
+#define LogWindowsError(thing) (void)0
+#else
+static void LogWindowsError(const char* file, int line, const char* thing) {
+#define LogWindowsError(thing) LogWindowsError(__FILE__, __LINE__, thing)
+    char s[256];
+    int e = GetLastError();
+    WinStrerror(e, s, sizeof(s));
+    fprintf(stderr, "%s:%d: error[%#x]: %s failed: %s\n", file, line, e, thing, s);
+}
+#endif
+
+static void *Recalloc(void *ptr, uint64_t newSize) {
+    HANDLE heap = GetProcessHeap();
+    if (!ptr) {
+        return HeapAlloc(heap, HEAP_ZERO_MEMORY, newSize);
+    }
+    if (!newSize) {
+        HeapFree(heap, 0, ptr);
+        return 0;
+    }
+    return HeapReAlloc(heap, HEAP_ZERO_MEMORY, ptr, newSize);
+}
+
+uint64_t WinSeek(int fd, uint64_t offset, int whence) {
+    HANDLE hFile;
+    DWORD winwhence;
+    LARGE_INTEGER distanceToMove;
+    LARGE_INTEGER newFilePointer;
+    distanceToMove.QuadPart = offset;
+    switch (whence) {
+    case SEEK_SET:
+        winwhence = FILE_BEGIN;
+        break;
+    case SEEK_CUR:
+        winwhence = FILE_CURRENT;
+        break;
+    case SEEK_END:
+        winwhence = FILE_END;
+        break;
+    default:
+        LogError("bad lseek() whence");
+        errno = EINVAL;
+        return -1;
+    }
+    hFile = (HANDLE)_get_osfhandle(fd);
+    if (hFile == INVALID_HANDLE_VALUE) {
+        LogWindowsError("_get_osfhandle");
+        errno = EBADF;
+        return -1;
+    }
+    if (GetFileType(hFile) != FILE_TYPE_DISK) {
+        LogError("bad file type for lseek()");
+        errno = ESPIPE;
+        return -1;
+    }
+    if (!SetFilePointerEx(hFile, distanceToMove, &newFilePointer, winwhence)) {
+        LogWindowsError("SetFilePointerEx");
+        errno = EPERM;
+        return -1;
+    }
+    return newFilePointer.QuadPart;
+}
+
+int WinFtruncate(int fd, uint64_t length) {
+    HANDLE hFile;
+    LARGE_INTEGER old, neu;
+    hFile = (HANDLE)_get_osfhandle(fd);
+    if (hFile == INVALID_HANDLE_VALUE) {
+        LogWindowsError("_get_osfhandle");
+        errno = EBADF;
+        return -1;
+    }
+    // save current file position
+    old.QuadPart = 0;
+    neu.QuadPart = 0;
+    if (!SetFilePointerEx(hFile, neu, &old, FILE_CURRENT)) {
+        LogWindowsError("SetFilePointerEx#1");
+        return -1;
+    }
+    // set current position to new file size
+    neu.QuadPart = length;
+    if (!SetFilePointerEx(hFile, neu, NULL, FILE_BEGIN)) {
+        LogWindowsError("SetFilePointerEx#2");
+        return -1;
+    }
+    // change the file size
+    if (!SetEndOfFile(hFile)) {
+        LogWindowsError("SetEndOfFile");
+        SetFilePointerEx(hFile, old, NULL, FILE_BEGIN);
+        return -1;
+    }
+    // restore the original file position
+    // win32 allows this to exceed the end of file
+    if (!SetFilePointerEx(hFile, old, NULL, FILE_BEGIN)) {
+        LogWindowsError("SetFilePointerEx>3");
+        return -1;
+    }
+    return 0;
+}
+
+int WinMadvise(void *addr, uintptr_t length, int advice) {
+    switch (advice) {
+    case MADV_NORMAL:
+    case MADV_DONTNEED:
+    case MADV_SEQUENTIAL:
+        return 0;
+    case MADV_RANDOM:
+    case MADV_WILLNEED: {
+        HANDLE proc;
+        WIN32_MEMORY_RANGE_ENTRY entry;
+        proc = GetCurrentProcess();
+        entry.VirtualAddress = addr;
+        entry.NumberOfBytes = length;
+        if (!PrefetchVirtualMemory(proc, 1, &entry, 0)) {
+            LogWindowsError("PrefetchVirtualMemory");
+            errno = ENOMEM;
+            return -1;
+        }
+        return 0;
+    }
+    default:
+        errno = EINVAL;
+        return -1;
+    }
+}
+
+int WinUnmap(void *addr, uintptr_t length) {
+    void *view;
+    HANDLE hand;
+    HANDLE fand;
+    int i, err = 0;
+    uintptr_t a, b;
+    uintptr_t x, y;
+    // compute the requested interval
+    // 1. length can't be zero
+    // 2. length is rounded up to the page size
+    // 3. addr must be aligned to page boundary
+    a = (uintptr_t)addr;
+    b = a + Roundup(length, 65536);
+    if (!length) {
+        LogError("tried to munmap zero bytes");
+        errno = EINVAL;
+        return -1;
+    }
+    if (a & 65535) {
+        LogError("tried to munmap an address that's not 64kb aligned");
+        errno = EINVAL;
+        return -1;
+    }
+    // 1. we permit unmapping multiple maps in one call
+    // 2. we don't care if the matched mappings aren't contiguous
+    // 3. it's an error if a matched mapping only partially overlaps
+    // 4. similar to close() we release all resources possible on error
+    Lock();
+    for (i = 0; i < g_winmaps.n; ++i) {
+        if (!g_winmaps.p[i].hand) {
+            // this array slot is empty
+            continue;
+        }
+        // compute overlap between known mapping and requested interval
+        x = Max(a, g_winmaps.p[i].addr);
+        y = Min(b, g_winmaps.p[i].addr + g_winmaps.p[i].length);
+        if (x >= y) {
+            // there isn't any overlap
+            continue;
+        }
+        if (y - x != g_winmaps.p[i].length) {
+            // requested interval partially overlapped this mapping
+            // therefore we can't unmap it and must report an error
+            LogError("tried to partially unmap a mapping");
+            err = ENOMEM;
+            continue;
+        }
+        // save the information we care about
+        view = (void *)g_winmaps.p[i].addr;
+        hand = g_winmaps.p[i].hand;
+        fand = g_winmaps.p[i].fand;
+        // delete this mapping from the global array
+        g_winmaps.p[i].hand = 0;
+        // perform the systems operations
+        // safe to release lock since g_winmaps.n is monotonic
+        Unlock();
+        if (!UnmapViewOfFile(view)) {
+            LogWindowsError("UnmapViewOfFile");
+        }
+        if (!CloseHandle(hand)) {
+            LogWindowsError("CloseHandle#1");
+        }
+        if (fand != INVALID_HANDLE_VALUE) {
+            if (!CloseHandle(fand)) {
+                LogWindowsError("CloseHandle#2");
+            }
+        }
+        Lock();
+    }
+    Unlock();
+    if (err) {
+        errno = err;
+        return -1;
+    }
+    return 0;
+}
+
+void* WinMap(void *addr, uintptr_t length, int prot, int flags, int fd, uint64_t offset) {
+    int i;
+    LPVOID res;
+    HANDLE hand;
+    HANDLE hFile;
+    DWORD access;
+    DWORD wiprot;
+    uintptr_t fsize;
+    if (!length) {
+        LogError("mmap(length) was zero");
+        errno = EINVAL;
+        return MAP_FAILED;
+    }
+    length = Roundup(length, 65536);
+    if ((uintptr_t)addr & 65535) {
+        if (~flags & MAP_FIXED) {
+            addr = 0;
+        } else {
+            LogError("MAP_FIXED used with address that's not 64kb aligned");
+            errno = EINVAL;
+            return MAP_FAILED;
+        }
+    }
+    // these are the logical flag equivalents for creating mappings.  please
+    // note that any subsequent virtualprotect calls must be a subset of the
+    // permissions we're using here.  that's not a supported use case for us
+    if (flags & MAP_PRIVATE) {
+        // private mapping
+        if (prot & PROT_EXEC) {
+            if (prot & PROT_WRITE) {
+                if (flags & MAP_ANONYMOUS) {
+                    wiprot = PAGE_EXECUTE_READWRITE;
+                    access = FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE;
+                } else {
+                    wiprot = PAGE_EXECUTE_WRITECOPY;
+                    access = FILE_MAP_COPY | FILE_MAP_EXECUTE;
+                }
+            } else {
+                wiprot = PAGE_EXECUTE_READ;
+                access = FILE_MAP_READ | FILE_MAP_EXECUTE;
+            }
+        } else if (prot & PROT_WRITE) {
+            if (flags & MAP_ANONYMOUS) {
+                wiprot = PAGE_READWRITE;
+                access = FILE_MAP_READ | FILE_MAP_WRITE;
+            } else {
+                wiprot = PAGE_WRITECOPY;
+                access = FILE_MAP_COPY;
+            }
+        } else {
+            wiprot = PAGE_READONLY;
+            access = FILE_MAP_READ;
+        }
+    } else {
+        // shared mapping
+        if (prot & PROT_EXEC) {
+            if (prot & PROT_WRITE) {
+                wiprot = PAGE_EXECUTE_READWRITE;
+                access = FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE;
+            } else {
+                wiprot = PAGE_EXECUTE_READ;
+                access = FILE_MAP_READ | FILE_MAP_EXECUTE;
+            }
+        } else if (prot & PROT_WRITE) {
+            wiprot = PAGE_READWRITE;
+            access = FILE_MAP_READ | FILE_MAP_WRITE;
+        } else {
+            wiprot = PAGE_READONLY;
+            access = FILE_MAP_READ;
+        }
+    }
+    if (flags & MAP_ANONYMOUS) {
+        hFile = INVALID_HANDLE_VALUE;
+        fsize = length;
+        offset = 0;
+    } else {
+        fsize = 0;
+        hFile = (HANDLE)_get_osfhandle(fd);
+        if (hFile == INVALID_HANDLE_VALUE) {
+            LogWindowsError("_get_osfhandle");
+            errno = EBADF;
+            return MAP_FAILED;
+        }
+        if (!DuplicateHandle(GetCurrentProcess(), hFile,
+                             GetCurrentProcess(), &hFile,
+                             0, FALSE, DUPLICATE_SAME_ACCESS)) {
+            LogWindowsError("DuplicateHandle");
+            errno = EBADF;
+            return MAP_FAILED;
+        }
+    }
+    if (flags & MAP_FIXED) {
+        if (!addr) {
+            // zero chance of microsoft letting us map the null page
+            if (hFile != INVALID_HANDLE_VALUE) {
+                CloseHandle(hFile);
+            }
+            errno = EINVAL;
+            return MAP_FAILED;
+        } else {
+            // blow away any existing mappings on requested interval
+            if (WinUnmap(addr, length) == -1) {
+                // can only happen if we partially overlap an existing mapping
+                assert(errno == ENOMEM);
+                if (hFile != INVALID_HANDLE_VALUE) {
+                    CloseHandle(hFile);
+                }
+                return MAP_FAILED;
+            }
+        }
+    }
+    hand = CreateFileMapping(hFile, 0, wiprot,
+                             (DWORD)(fsize >> 32),
+                             (DWORD)fsize,
+                             0);
+    if (!hand) {
+        LogWindowsError("CreateFileMapping");
+        if (hFile != INVALID_HANDLE_VALUE) {
+            CloseHandle(hFile);
+        }
+        errno = EPERM;
+        return MAP_FAILED;
+    }
+    res = MapViewOfFileEx(hand, access,
+                          (DWORD)(offset >> 32),
+                          (DWORD)offset,
+                          length, addr);
+    if (!res) {
+        LogWindowsError("MapViewOfFileEx");
+        if (hFile != INVALID_HANDLE_VALUE) {
+            CloseHandle(hFile);
+        }
+        CloseHandle(hand);
+        errno = EPERM;
+        return MAP_FAILED;
+    }
+    if (flags & MAP_FIXED) {
+        // this assertion could legitimately fail if two threads engage in a
+        // race to create a MAP_FIXED mapping at the same address and that's
+        // certainly not the kind of use case we're designed to support here
+        assert(res == addr);
+    }
+    // record our new mapping in the global array
+    Lock();
+    for (i = 0; i < g_winmaps.n; ++i) {
+        if (!g_winmaps.p[i].hand) {
+            // we found an empty slot
+            break;
+        }
+    }
+    if (i == g_winmaps.n) {
+        // we need to grow the array
+        // it's important to use kernel32 memory
+        // our malloc implementation depends on this
+        int n2;
+        struct WinMap *p2;
+        p2 = g_winmaps.p;
+        n2 = g_winmaps.n;
+        if (n2) {
+            n2 += n2 >> 1;
+        } else {
+            n2 = 7;
+        }
+        if ((p2 = (struct WinMap*)Recalloc(p2, n2 * sizeof(*p2)))) {
+            g_winmaps.p = p2;
+            g_winmaps.n = n2;
+        } else {
+            Unlock();
+            LogError("recalloc failed");
+            UnmapViewOfFile(res);
+            CloseHandle(hand);
+            if (hFile != INVALID_HANDLE_VALUE) {
+                CloseHandle(hFile);
+            }
+            errno = ENOMEM;
+            return MAP_FAILED;
+        }
+    }
+    g_winmaps.p[i].hand = hand;
+    g_winmaps.p[i].fand = hFile;
+    g_winmaps.p[i].addr = (uintptr_t)res;
+    g_winmaps.p[i].length = length;
+    Unlock();
+    return res;
+}
+
+int WinMsync(void *addr, uintptr_t length, int flags) {
+    int i, err;
+    HANDLE hand;
+    uintptr_t x, y;
+    if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) {
+        LogError("bad msync flags");
+        errno = EINVAL;
+        return -1;
+    }
+    // 1. we do nothing if length is zero (unlike win32 api)
+    // 2. the requested interval may envelop multiple known mappings
+    // 3. we don't care if those mappings aren't contiguous or a hole exists
+    // 4. the requested interval may specify a subrange of any given mapping
+    Lock();
+    for (err = i = 0; i < g_winmaps.n; ++i) {
+        if (!g_winmaps.p[i].hand) {
+            // this array slot is empty
+            continue;
+        }
+        // compute overlap between known mapping and requested interval
+        x = Max((uintptr_t)addr, g_winmaps.p[i].addr);
+        y = Min((uintptr_t)addr + length, g_winmaps.p[i].addr + g_winmaps.p[i].length);
+        if (x >= y) {
+            // there isn't any overlap
+            continue;
+        }
+        // it's safe to release lock temporarily, since g_winmaps.n is monotonic
+        // any race conditions in handle being deleted should be caught by win32
+        hand = g_winmaps.p[i].fand;
+        Unlock();
+        // ensure coherency and that filesystem flush *will* happen
+        if (!FlushViewOfFile((void*)x, y - x)) {
+            LogWindowsError("FlushViewOfFile");
+            err = EPERM;
+        }
+        if (flags & MS_SYNC) {
+            // ensure that filesystem flush *has* happened
+            if (!FlushFileBuffers(hand)) {
+                LogWindowsError("FlushFileBuffers");
+                err = EPERM;
+            }
+        }
+        Lock();
+    }
+    Unlock();
+    if (err) {
+        errno = err;
+        return -1;
+    }
+    return 0;
+}
+
+#else // NEED_*_MAP
+
+// this is a normal unix platform
+// add some content to this object so the apple linker doesn't whine
+int justine_mmap_module;
+
+#endif // NEED_*_MMAP

mmap.h

@@ -0,0 +1,150 @@
+#pragma once
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+#include <fcntl.h>
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined (_MSC_VER) && !(defined (_POSIX_MAPPED_FILES))
+#define NEED_WIN32_MMAP
+#include <Windows.h>
+#include <io.h>
+
+#ifndef PROT_READ
+#define PROT_READ 1
+#endif
+#ifndef PROT_WRITE
+#define PROT_WRITE 2
+#endif
+#ifndef PROT_EXEC
+#define PROT_EXEC 4
+#endif
+
+#ifndef MAP_SHARED
+#define MAP_SHARED 1
+#endif
+#ifndef MAP_PRIVATE
+#define MAP_PRIVATE 2
+#endif
+#ifndef MAP_FIXED
+#define MAP_FIXED 16
+#endif
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS 32
+#endif
+#ifndef MAP_FAILED
+#define MAP_FAILED ((void*)-1)
+#endif
+
+#ifndef O_RDONLY
+#define O_RDONLY _O_RDWR  // intentional smudge for mmap()
+#endif
+#ifndef O_WRONLY
+#define O_WRONLY _O_WRONLY
+#endif
+#ifndef O_RDWR
+#define O_RDWR _O_RDWR
+#endif
+#ifndef O_CREAT
+#define O_CREAT _O_CREAT
+#endif
+#ifndef O_TRUNC
+#define O_TRUNC _O_TRUNC
+#endif
+#ifndef O_EXCL
+#define O_EXCL _O_EXCL
+#endif
+
+#ifndef MADV_NORMAL
+#define MADV_NORMAL 0
+#endif
+#ifndef MADV_DONTNEED
+#define MADV_DONTNEED 4
+#endif
+#ifndef MADV_RANDOM
+#define MADV_RANDOM 1
+#endif
+#ifndef MADV_SEQUENTIAL
+#define MADV_SEQUENTIAL 2
+#endif
+#ifndef MADV_WILLNEED
+#define MADV_WILLNEED 3
+#endif
+
+#ifndef MS_ASYNC
+#define MS_ASYNC 1
+#endif
+#ifndef MS_INVALIDATE
+#define MS_INVALIDATE 2
+#endif
+#ifndef MS_SYNC
+#define MS_SYNC 4
+#endif
+
+#ifndef SEEK_SET
+#define SEEK_SET 0
+#endif
+#ifndef SEEK_CUR
+#define SEEK_CUR 1
+#endif
+#ifndef SEEK_END
+#define SEEK_END 2
+#endif
+
+#ifndef mmap
+#define mmap WinMap
+#endif
+#ifndef munmap
+#define munmap WinUnmap
+#endif
+#ifndef open
+#define open _open
+#endif
+#ifndef close
+#define close _close
+#endif
+#ifndef lseek
+#define lseek WinSeek
+#endif
+#ifndef msync
+#define msync WinMsync
+#endif
+#ifndef madvise
+#define madvise WinMadvise
+#endif
+#ifndef ftruncate
+#define ftruncate WinFtruncate
+#endif
+
+uint64_t WinSeek(int, uint64_t, int);
+int WinMsync(void *, uintptr_t, int);
+int WinMadvise(void *, uintptr_t, int);
+int WinFtruncate(int, uint64_t);
+int WinUnmap(void *, uintptr_t);
+void *WinMap(void *, uintptr_t, int, int, int, uint64_t);
+
+#else // _MSC_VER
+
+#include <unistd.h>
+#include <sys/mman.h>
+
+#ifndef MAP_ANONYMOUS
+#define NEED_POSIX_MMAP
+#define mmap PosixMmap
+#define MAP_ANONYMOUS 0x10000000
+void *PosixMmap(void*, size_t, int, int, int, off_t);
+#endif // MAP_ANONYMOUS
+
+#endif // _MSC_VER
+
+#ifdef __cplusplus
+}
+#endif