From 9ebc43e0727cfbf8e40e41517965544ddb054d18 Mon Sep 17 00:00:00 2001 From: dtookey Date: Wed, 20 Jul 2022 15:16:05 -0400 Subject: [PATCH] massive refactor. StaticRoute.go is still a mess. needs a big overhaul still. --- servlet/server.go | 42 ++---- servlet/staticRoute.go | 139 ++++++++++--------- tmp.go | 9 ++ {cacheutil => vinegarUtil}/WebLRU.go | 195 ++++++++++++++------------- vinegarUtil/files.go | 65 +++++++++ 5 files changed, 257 insertions(+), 193 deletions(-) create mode 100644 tmp.go rename {cacheutil => vinegarUtil}/WebLRU.go (55%) create mode 100644 vinegarUtil/files.go diff --git a/servlet/server.go b/servlet/server.go index 46ea1c1..49e70d3 100644 --- a/servlet/server.go +++ b/servlet/server.go @@ -1,19 +1,19 @@ package servlet import ( - "fmt" "html/template" "io/ioutil" "net/http" "os" "regexp" - "vinegar/cacheutil" + "vinegar/vinegarUtil" ) const ( defaultLruSize = int64(1024 * 1024 * 50) ContentTypeHeaderKey = "Content-Type" ContentEncodingHeaderKey = "Content-Encoding" + AcceptEncodingHeaderKey = "Accept-Encoding" ) type ErrorResponse struct { @@ -25,32 +25,28 @@ type ( VinegarServlet struct { Port string Routes []*VinegarRoute - Cache *cacheutil.Lru } VinegarRoute struct { Pattern *regexp.Regexp Handler VinegarHandlerFunction + Cache vinegarUtil.Cache } VinegarHandlerFunction func(w http.ResponseWriter, req *http.Request) ) func NewServlet(port string) *VinegarServlet { - lru := cacheutil.NewLRU(defaultLruSize) - srv := VinegarServlet{Port: port, Cache: lru} + srv := VinegarServlet{Port: port} return &srv } func NewServletRoute(routePattern string, handleFunc VinegarHandlerFunction) *VinegarRoute { - route := VinegarRoute{} pattern := regexp.MustCompile(routePattern) - route.Pattern = pattern - route.Handler = handleFunc - + route := VinegarRoute{Pattern: pattern, Handler: handleFunc, Cache: vinegarUtil.NewLRU(defaultLruSize)} return &route } @@ -60,7 +56,6 @@ func (s *VinegarServlet) AddRoute(route *VinegarRoute) { func (s *VinegarServlet) ServeHTTP(w http.ResponseWriter, req *http.Request) { path := req.URL.Path - fmt.Println("Attempting to match request for " + path) for _, route := range s.Routes { if route.Pattern.MatchString(path) { route.Handler(w, req) @@ -69,7 +64,11 @@ func (s *VinegarServlet) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } -func StartServerGood(s *VinegarServlet) { +func (s *VinegarServlet) Start() { + if len(s.Routes) < 1 { + + } + err := http.ListenAndServe(s.Port, s) if err != nil { @@ -105,24 +104,3 @@ func SendError(w http.ResponseWriter, code int, msg string) { panic(err) } } - -func renderTemplate(w http.ResponseWriter, pathlike string, data any) { - templateHelper := template.New(pathlike) - f, err := os.OpenFile(pathlike, os.O_RDONLY, 0777) - if err != nil { - panic(err) - } - defer f.Close() - - content, err := ioutil.ReadAll(f) - if err != nil { - panic(err) - } - - templ, err := templateHelper.Parse(string(content)) - err = templ.Execute(w, data) - if err != nil { - panic(err) - } - -} diff --git a/servlet/staticRoute.go b/servlet/staticRoute.go index 6ad9f70..e34c460 100644 --- a/servlet/staticRoute.go +++ b/servlet/staticRoute.go @@ -1,96 +1,107 @@ package servlet import ( - "fmt" "net/http" "path" "strings" - "vinegar/cacheutil" + util "vinegar/vinegarUtil" ) type ( FileRoute struct { *VinegarRoute + srv *VinegarServlet fileRoot string + UseCache bool } ) -func NewImageRoute(urlPattern string, pathlike string, servlet *VinegarServlet) *FileRoute { +func NewImageRoute(servlet *VinegarServlet, urlPattern string, pathlike string, useCache bool) *FileRoute { defaultPrune := strings.Replace(urlPattern, ".*", "", -1) - rootRoute := NewServletRoute(urlPattern, createUncompressedFileServletFunction(defaultPrune, pathlike, servlet.Cache)) - imgRoute := FileRoute{rootRoute, pathlike} + imgRoute := FileRoute{srv: servlet, fileRoot: pathlike, UseCache: useCache} + rootRoute := NewServletRoute(urlPattern, createUncompressedFileServletFunction(&imgRoute, defaultPrune, pathlike)) + imgRoute.VinegarRoute = rootRoute //i *kinda* don't like this pattern return &imgRoute } -func NewTextRoute(urlPattern string, pathlike string, servlet *VinegarServlet) *FileRoute { +func NewTextRoute(servlet *VinegarServlet, urlPattern string, pathlike string, useCache bool) *FileRoute { defaultPrune := strings.Replace(urlPattern, ".*", "", -1) - rootRoute := NewServletRoute(urlPattern, createCompressibleFileServletFunction(defaultPrune, pathlike, servlet.Cache)) - fr := FileRoute{rootRoute, pathlike} + fr := FileRoute{srv: servlet, fileRoot: pathlike, UseCache: useCache} + textRouteHandler := createCompressibleFileServletFunction(&fr, defaultPrune, pathlike) + rootRoute := NewServletRoute(urlPattern, textRouteHandler) //i *still* kinda don't like this pattern + fr.VinegarRoute = rootRoute + return &fr } -func NewSingleFileRoute(urlPattern string, pathlike string, servlet *VinegarServlet) *FileRoute { - fun := createSingleFileServletFunction(pathlike) +func NewSingleFileRoute(servlet *VinegarServlet, urlPattern string, pathlike string, useCache bool) *FileRoute { route := FileRoute{ - VinegarRoute: NewServletRoute("^"+urlPattern+"$", fun), - fileRoot: pathlike, + srv: servlet, + fileRoot: pathlike, + UseCache: useCache, } + singleFileServletHandler := createSingleFileServletFunction(&route) + sfCache := util.NewSingleFileCache(pathlike) + + parentRoute := NewServletRoute(urlPattern, singleFileServletHandler) + parentRoute.Handler = singleFileServletHandler + parentRoute.Cache = sfCache + + route.VinegarRoute = parentRoute + return &route } -func createSingleFileServletFunction(pathlike string) VinegarHandlerFunction { - cache := cacheutil.NewSingleFileCache(pathlike) - var fun VinegarHandlerFunction = func(w http.ResponseWriter, req *http.Request) { - acceptsGzip := clientAcceptsGzip(req) +func createSingleFileServletFunction(route *FileRoute) VinegarHandlerFunction { - if cacheutil.UseCache { - var content []byte - if acceptsGzip { - w.Header().Add(ContentEncodingHeaderKey, "gzip") - content = *cache.CompressedContent - } else { - content = *cache.Content - } - w.Header().Add(ContentTypeHeaderKey, cache.Mimetype) - _, err := w.Write(content) - if err != nil { - panic(err) - } + var fun VinegarHandlerFunction = func(w http.ResponseWriter, req *http.Request) { + var cache *util.LruEntry + var exists bool + + if route.UseCache { + cache, exists = route.Cache.Get("") } else { - contents, exists := cacheutil.GetDiskContent(pathlike) - if exists { - fmt.Printf("returning contents: len of %d ", len(*contents)) - w.Header().Add(ContentTypeHeaderKey, cache.Mimetype) - _, err := w.Write(*contents) - if err != nil { - panic(err) - } - return - } else { - w.WriteHeader(404) - w.Write(nil) - fmt.Println("returning nothing") - return - } + cache, exists = route.Cache.GetFresh("") + } + + if !exists { + SendError(w, 404, "File not found.") + return + } + + var content []byte + if clientAcceptsGzip(req) { + content = cache.CompressedContent + w.Header().Add(ContentEncodingHeaderKey, "gzip") + } else { + content = cache.Content + } + + _, err := w.Write(content) + if err != nil { + panic(err) } } return fun } -func createCompressibleFileServletFunction(basePattern string, pathlike string, cache *cacheutil.Lru) VinegarHandlerFunction { +func createCompressibleFileServletFunction(route *FileRoute, basePattern string, pathlike string) VinegarHandlerFunction { var fun VinegarHandlerFunction = func(w http.ResponseWriter, req *http.Request) { stub := strings.Replace(req.URL.Path, basePattern, "", 1) - cachedContent, exists := cache.Get(stub) + cachedContent, exists := route.Cache.Get(stub) //i don't like this logic below. we need to streamline this a lot better. it's a twisty jungle right now + + resourcePath := path.Join(pathlike, stub) + if !exists { - content, fileExists := cacheutil.GetDiskContent(path.Join(pathlike, stub)) + content, fileExists := util.GetDiskContent(resourcePath) if fileExists { - if cacheutil.UseCache { - cache.Put(stub, content) - cachedContent, _ = cache.Get(stub) + if route.UseCache { + route.Cache.Put(stub, resourcePath) + cachedContent, _ = route.Cache.Get(stub) } else { - w.Header().Add(ContentTypeHeaderKey, cacheutil.GuessMimetype(stub)) + w.Header().Add(ContentTypeHeaderKey, util.GuessMimetype(stub)) w.Write(*content) return } @@ -116,20 +127,17 @@ func createCompressibleFileServletFunction(basePattern string, pathlike string, return fun } -func createUncompressedFileServletFunction(basePattern string, pathlike string, cache *cacheutil.Lru) VinegarHandlerFunction { +func createUncompressedFileServletFunction(route *FileRoute, basePattern string, pathlike string) VinegarHandlerFunction { var fun VinegarHandlerFunction = func(w http.ResponseWriter, req *http.Request) { stub := strings.Replace(req.URL.Path, basePattern, "", 1) - - entry, exists := cache.Get(stub) + resourcePath := path.Join(pathlike, stub) + entry, exists := route.Cache.Get(stub) if !exists { - fileContent, fExists := cacheutil.GetDiskContent(path.Join(pathlike, stub)) - if fExists { - cache.Put(stub, fileContent) - entry, exists = cache.Get(stub) - } else { - SendError(w, 404, "Oops! Something went wrong!") - return - } + route.Cache.Put(stub, resourcePath) + entry, exists = route.Cache.Get(stub) + } else { + SendError(w, 404, "Oops! Something went wrong!") + return } if !exists { @@ -146,9 +154,6 @@ func createUncompressedFileServletFunction(basePattern string, pathlike string, } func clientAcceptsGzip(req *http.Request) bool { - encodings := req.Header.Get("Accept-Encoding") - if strings.Contains(encodings, "gzip") { - return true - } - return false + encodings := req.Header.Get(AcceptEncodingHeaderKey) + return strings.Contains(encodings, "gzip") } diff --git a/tmp.go b/tmp.go new file mode 100644 index 0000000..6d03aec --- /dev/null +++ b/tmp.go @@ -0,0 +1,9 @@ +package main + +import "vinegar/servlet" + +func main() { + serv := servlet.NewServlet(":8080") + + serv.Start() +} diff --git a/cacheutil/WebLRU.go b/vinegarUtil/WebLRU.go similarity index 55% rename from cacheutil/WebLRU.go rename to vinegarUtil/WebLRU.go index 7d508aa..4d1a868 100644 --- a/cacheutil/WebLRU.go +++ b/vinegarUtil/WebLRU.go @@ -1,11 +1,8 @@ -package cacheutil +package vinegarUtil import ( - "bytes" - gzip2 "compress/gzip" + "errors" "fmt" - "io/ioutil" - "os" "strings" "time" ) @@ -14,10 +11,6 @@ const ( DefaultCacheTimeInMinutes = 15 ) -var ( - UseCache = true -) - type ( lru map[string]*LruEntry Lru struct { @@ -27,6 +20,7 @@ type ( } LruEntry struct { + path string Content []byte CompressedContent []byte created time.Time @@ -37,11 +31,15 @@ type ( } SingleFileCache struct { - Content *[]byte - CompressedContent *[]byte - timeCreated time.Time - timeExpires time.Time - Mimetype string + path string + entry *LruEntry + } + + Cache interface { + Get(key string) (*LruEntry, bool) + GetFresh(key string) (*LruEntry, bool) + Put(key string, pathlike string) error + Remove(key string) } ) @@ -51,59 +49,13 @@ func NewLRU(size int64) *Lru { } func NewSingleFileCache(pathlike string) *SingleFileCache { - bits, exists := GetDiskContent(pathlike) - if !exists { - panic("Could not load single file for single file path: " + pathlike) - } - compressedBits := GZipBytes(bits) - - cache := SingleFileCache{ - Content: bits, - CompressedContent: compressedBits, - timeCreated: time.Now(), - timeExpires: time.Now().Add(DefaultCacheTimeInMinutes), - Mimetype: GuessMimetype(pathlike), - } - return &cache -} - -func (l *Lru) Put(key string, content *[]byte) { - entry, exists := (*l.entries)[key] - - zippedContent := *GZipBytes(content) - size := int64(len(*content)) + int64(len(zippedContent)) - - l.ensureVacancy(size) - - if exists { - entry.Content = *content - entry.CompressedContent = zippedContent - entry.created = time.Now() - entry.expires = time.Now().Add(DefaultCacheTimeInMinutes * time.Minute) - entry.mostRecentAccess = time.Now() - entry.totalSize = size - } else { - mimetype := GuessMimetype(key) - entry = &LruEntry{ - Content: *content, - CompressedContent: zippedContent, - created: time.Now(), - expires: time.Now().Add(DefaultCacheTimeInMinutes * time.Minute), - mostRecentAccess: time.Now(), - totalSize: size, - MimeType: mimetype, - } - } - - (*l.entries)[key] = entry - l.currentSize = l.currentSize + size + entry := newLRUEntry(pathlike) + sfc := SingleFileCache{path: pathlike, entry: entry} + return &sfc } func (l *Lru) Get(key string) (*LruEntry, bool) { - if !UseCache { - return nil, false - } entry, exists := (*l.entries)[key] if exists && entry.expires.Before(time.Now()) { fmt.Println("Cache miss due to expired content") @@ -113,12 +65,48 @@ func (l *Lru) Get(key string) (*LruEntry, bool) { if exists { entry.mostRecentAccess = time.Now() } else { - fmt.Printf("cacheutil miss for '%s'\n", key) + fmt.Printf("vinegarUtil miss for '%s'\n", key) } return entry, exists } } +func (l *Lru) GetFresh(key string) (*LruEntry, bool) { + l.Remove(key) + return l.Get(key) +} + +func (l *Lru) Put(key string, pathlike string) error { + entry, exists := (*l.entries)[key] + + var size int64 + + if exists { + content, fExists := GetDiskContent(pathlike) + if !fExists { + return errors.New("attempted to refresh cache with file that no longer exists on disk") + } + zippedContent := *GZipBytes(content) + size = int64(len(*content)) + int64(len(zippedContent)) + l.ensureVacancy(size) + + entry.Content = *content + entry.CompressedContent = zippedContent + entry.created = time.Now() + entry.expires = time.Now().Add(DefaultCacheTimeInMinutes * time.Minute) + entry.mostRecentAccess = time.Now() + entry.totalSize = size + } else { + entry = newLRUEntry(pathlike) + + } + + (*l.entries)[key] = entry + + l.recalcSize() + return nil +} + func (l *Lru) Remove(key string) { entry, exists := (*l.entries)[key] if exists { @@ -159,19 +147,57 @@ func (l *Lru) getLeastRecentlyUsedKey() string { return leastUsedKey } -func GZipBytes(uncompressed *[]byte) *[]byte { - buff := bytes.Buffer{} - gzip := gzip2.NewWriter(&buff) - _, err := gzip.Write(*uncompressed) - if err != nil { - panic(err) +func (l *Lru) recalcSize() { + var total int64 + for _, entry := range *l.entries { + total += entry.totalSize } - err = gzip.Flush() - if err != nil { - panic(err) + l.currentSize = total +} + +func (sfc *SingleFileCache) Get(key string) (*LruEntry, bool) { + return sfc.entry, true +} + +func (sfc *SingleFileCache) GetFresh(key string) (*LruEntry, bool) { + entry := newLRUEntry(sfc.entry.path) + sfc.entry = entry + return entry, true +} + +// Put THIS WILL DELETE ANYTHING YOU HAVE STORED IN THIS CACHE. +func (sfc *SingleFileCache) Put(key string, pathlike string) error { + //there's a bug in this. we don't return an error from newLRUEntry, so if the file disappears, the server will die + entry := newLRUEntry(pathlike) + sfc.entry = entry + return nil +} + +func (sfc *SingleFileCache) Remove(key string) { + //i'm actually not sure what to do here. why would you ever remove from a single-file cache? +} + +func newLRUEntry(pathlike string) *LruEntry { + bits, exists := GetDiskContent(pathlike) + if !exists { + panic("Could not load single file for single file path: " + pathlike) } - compressed := buff.Bytes() - return &compressed + + compressedBits := GZipBytes(bits) + + size := int64(len(*bits)) + int64(len(*compressedBits)) + + entry := LruEntry{ + path: pathlike, + Content: *bits, + CompressedContent: *compressedBits, + created: time.Now(), + expires: time.Now().Add(DefaultCacheTimeInMinutes * time.Minute), + mostRecentAccess: time.Now(), + totalSize: size, + MimeType: GuessMimetype(pathlike), + } + return &entry } func GuessMimetype(filePath string) string { @@ -197,22 +223,3 @@ func GuessMimetype(filePath string) string { } return "application/octet-stream" } - -func GetDiskContent(filePath string) (*[]byte, bool) { - _, ferr := os.Stat(filePath) - if os.IsNotExist(ferr) { - return nil, false - } else { - f, err := os.OpenFile(filePath, os.O_RDONLY, 0755) - if err != nil { - panic(err) - } - defer f.Close() - content, err := ioutil.ReadAll(f) - - if err != nil { - panic(err) - } - return &content, true - } -} diff --git a/vinegarUtil/files.go b/vinegarUtil/files.go new file mode 100644 index 0000000..f34d4c8 --- /dev/null +++ b/vinegarUtil/files.go @@ -0,0 +1,65 @@ +package vinegarUtil + +import ( + "bytes" + gzip2 "compress/gzip" + "html/template" + "io/ioutil" + "net/http" + "os" +) + +func RenderTemplate(w http.ResponseWriter, pathlike string, data any) { + templateHelper := template.New(pathlike) + f, err := os.OpenFile(pathlike, os.O_RDONLY, 0777) + if err != nil { + panic(err) + } + defer f.Close() + + content, err := ioutil.ReadAll(f) + if err != nil { + panic(err) + } + + templ, err := templateHelper.Parse(string(content)) + err = templ.Execute(w, data) + if err != nil { + panic(err) + } + +} + +func GetDiskContent(filePath string) (*[]byte, bool) { + _, ferr := os.Stat(filePath) + if os.IsNotExist(ferr) { + return nil, false + } else { + f, err := os.OpenFile(filePath, os.O_RDONLY, 0755) + if err != nil { + panic(err) + } + defer f.Close() + content, err := ioutil.ReadAll(f) + + if err != nil { + panic(err) + } + return &content, true + } +} + +func GZipBytes(uncompressed *[]byte) *[]byte { + buff := bytes.Buffer{} + gzip := gzip2.NewWriter(&buff) + _, err := gzip.Write(*uncompressed) + if err != nil { + panic(err) + } + err = gzip.Flush() + if err != nil { + panic(err) + } + compressed := buff.Bytes() + return &compressed +}