diff options
author | Ted Unangst <tedu@tedunangst.com> | 2022-03-01 01:34:44 -0500 |
---|---|---|
committer | Ted Unangst <tedu@tedunangst.com> | 2022-03-01 01:34:44 -0500 |
commit | ca68ff1fd759be5b875da9908aedd585442cb8d8 (patch) | |
tree | 01a7d6870d616643096e012b43d846a99044e938 | |
parent | c2f7d512c5580dc7a184dd119b9071d3d3f45a42 (diff) |
start reworking to use yaegi
-rw-r--r-- | filter.go | 33 | ||||
-rw-r--r-- | filter.lua | 143 | ||||
-rw-r--r-- | go.mod | 4 | ||||
-rw-r--r-- | go.sum | 9 | ||||
-rw-r--r-- | interpreter.go | 39 | ||||
-rw-r--r-- | luafunctions.go | 95 | ||||
-rw-r--r-- | luainterface.go | 183 | ||||
-rw-r--r-- | miniwebproxy.go | 27 |
8 files changed, 64 insertions, 469 deletions
diff --git a/filter.go b/filter.go index 8286a65..4dc90a3 100644 --- a/filter.go +++ b/filter.go @@ -293,10 +293,8 @@ func clean(w io.Writer, node *html.Node, baseurl *url.URL) { } } -func filter(L *Interpreter, w io.Writer, resp *http.Response, req *http.Request) { - filtresult := L.Call("respfilter", req, resp, w) - if filtresult != "" { - log.Printf("respfilter says %s", filtresult) +func filter(interp *interpreter, w io.Writer, resp *http.Response, req *http.Request) { + if interp.FilterResponse(w, req, resp) { return } if resp.StatusCode != 200 { @@ -331,7 +329,7 @@ func filter(L *Interpreter, w io.Writer, resp *http.Response, req *http.Request) gzw = bufio.NewWriter(respwriter) } - go dofiltering(L, respwriter, gzw, reader, req) + go dofiltering(interp, respwriter, gzw, reader, req) resp.Header.Set("Content-Type", "text/html; charset=utf-8") resp.Header.Del("Content-Length") @@ -348,7 +346,7 @@ type flushWriter interface { Flush() error } -func dofiltering(L *Interpreter, under io.WriteCloser, w flushWriter, reader io.Reader, +func dofiltering(interp *interpreter, under io.WriteCloser, w flushWriter, reader io.Reader, req *http.Request) { defer under.Close() wcloser, ok := w.(io.Closer) @@ -364,24 +362,7 @@ func dofiltering(L *Interpreter, under io.WriteCloser, w flushWriter, reader io. return } - filtresult := L.Call("htmlfilter", w, req, req.URL.Hostname(), req.URL.Path, root) - if filtresult == "" { - log.Printf("did not rewrite %s, raw rendering", req.URL.String()) - html.Render(w, root) - return - } -} - -func prefilter(L *Interpreter, req *http.Request) { - L.Call("prefilter", req) -} - -func shouldintercept(L *Interpreter, host string) bool { - filtresult := L.Call("shouldfilter", host) - if filtresult != "" { - return true - } - return false + interp.FilterHTML(w, req, root) } func filteronefile(fname string) { @@ -398,7 +379,7 @@ func filteronefile(fname string) { log.Fatal(err) } resp.Body = fd - L := getinterpreter() - filter(L, os.Stdout, &resp, &req) + interp := getinterpreter() + filter(interp, os.Stdout, &resp, &req) fmt.Printf("\n") } diff --git a/filter.lua b/filter.lua deleted file mode 100644 index 7cc7256..0000000 --- a/filter.lua +++ /dev/null @@ -1,143 +0,0 @@ - -local headsel = cssfilter("head") -local headersel = cssfilter("header.post__header") - -local articlesels = { - cssfilter("main"), - cssfilter("article"), - cssfilter("div#main"), - cssfilter("#main-article"), - cssfilter(".main-content"), - cssfilter("#body"), - cssfilter("#content"), - cssfilter(".content"), - cssfilter("div#article"), - cssfilter("div.article"), - cssfilter("div.post"), - cssfilter("div.post-outer"), - cssfilter(".l-root"), - cssfilter(".content-container"), -} -local annoyances = { - cssfilter("div.zipr-recirc"), - cssfilter("section#news-feed"), - cssfilter("div#comments"), - cssfilter(".visually-hidden"), - cssfilter(".newsletter-signup"), -} - -function newconnection(req) -end - -function endconnection() -end - -function shouldfilter(host) - if host:match("github.com") then return "" end - if host:match("amazon.com") then return "" end - if host:match("twitter.com") then return "" end - if host:match("cdn.googlesource.com") then return "" end - return "y" -end - -function prefilter(req) - local host = reqgethost(req) - host = host:gsub("www.reddit.com", "old.reddit.com") - reqsethost(req, host) -end - -local dislikeCF = false - -function respfilter(req, resp, outw) - if dislikeCF then - local serv = respgetheader(resp, "Server") - if serv == "cloudflare" then - writestring(outw, "HTTP/1.1 302 Moved Temporarily\r\n") - writestring(outw, "Location: https://web.archive.org/save/" .. reqgethost(req) .. - reqgetpath(req) .. "\r\n\r\n") - return "done" - end - end -end - -function htmlfilter(outw, req, host, path, root) - print("trying to filter in lua", host, path) - if true and path == "/" then - return "" - end - --savehtml("root.html", root) - local article - for i = 1, #articlesels do - article = matchall(root, articlesels[i]) - if #article == 1 then - break - end - end - if #article ~= 1 then - print("didn't find anything to grab") - return "" - end - article = article[1] - - --savehtml("saved.html", article) - - writestring(outw, prolog) - local head = matchfirst(root, headsel) - if head then - clean(outw, head, req) - end - writestring(outw, "</head>\n<body>\n") - local header = matchfirst(root, headersel) - if header then - clean(outw, header, req) - end - if host == "www.washingtonpost.com" then - local headlinesel = cssfilter("#topper-headline-wrapper") - local headline = matchfirst(root, headlinesel) - if headline then - clean(outw, headline, req) - end - end - if host:match("blogspot") then - article = root - end - if host:match("livejournal") then - local sel = cssfilter("div.box") - local entries = matchall(article, sel) - for i = 2, #entries do - removenode(entries[i]) - end - end - for i = 1, #annoyances do - local bad = matchfirst(article, annoyances[i]) - if bad then - removenode(bad) - end - end - - clean(outw, article, req) - print("i did it, i cleaned it") - return "cleaned" -end - -prolog = [==[ -<!doctype html> -<html> -<head> -<meta charset="utf-8"> -<style> -body { - max-width: 940px; - margin: auto; - font-size: 1.4em; - line-height: 1.5; -} -img { - max-width: 900px; -} -pre { - white-space: pre-wrap; - word-wrap: break-word; -} -</style> -]==] diff --git a/go.mod b/go.mod index 2b47a8a..c5268e1 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,9 @@ module humungus.tedunangst.com/r/miniwebproxy +go 1.17 + require ( github.com/andybalholm/cascadia v1.0.0 - github.com/yuin/gopher-lua v0.0.0-20190206043414-8bfc7677f583 + github.com/traefik/yaegi v0.11.2 golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 ) diff --git a/go.sum b/go.sum index 1815a3d..f5e2ff8 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,10 @@ github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/yuin/gopher-lua v0.0.0-20190206043414-8bfc7677f583 h1:SZPG5w7Qxq7bMcMVl6e3Ht2X7f+AAGQdzjkbyOnNNZ8= -github.com/yuin/gopher-lua v0.0.0-20190206043414-8bfc7677f583/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +github.com/traefik/yaegi v0.11.2 h1:zosveTf5iIa60fAeQpaH4719b+bnlgsOvO7Nb/OTMTo= +github.com/traefik/yaegi v0.11.2/go.mod h1:RuCwD8/wsX7b6KoQHOaIFUfuH3gQIK4KWnFFmJMw5VA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU= golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/interpreter.go b/interpreter.go new file mode 100644 index 0000000..bb4cb92 --- /dev/null +++ b/interpreter.go @@ -0,0 +1,39 @@ +package main + +import ( + "io" + "net/http" + + "github.com/traefik/yaegi/interp" + "golang.org/x/net/html" +) + +type interpreter struct { + interp *interp.Interpreter +} + +func (interp *interpreter) NewConnection(r *http.Request) { +} + +func (interp *interpreter) ShouldIntercept(hostname string) bool { + return false +} + +func (interp *interpreter) Prefilter(req *http.Request) { +} + +func (interp *interpreter) FilterResponse(w io.Writer, req *http.Request, servresp *http.Response) bool { + return false +} + +func (interp *interpreter) FilterHTML(w io.Writer, req *http.Request, root *html.Node) { +} + +func getinterpreter() *interpreter { + i := new(interpreter) + i.interp = interp.New(interp.Options{}) + return nil +} + +func putinterpreter(interp *interpreter) { +} diff --git a/luafunctions.go b/luafunctions.go deleted file mode 100644 index a2fd4d0..0000000 --- a/luafunctions.go +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2017 Ted Unangst <tedu@tedunangst.com> - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -package main - -import ( - "io" - "net/http" - "os" - - "github.com/andybalholm/cascadia" - "github.com/yuin/gopher-lua" - "golang.org/x/net/html" -) - -func lcssfilter(css string) cascadia.Selector { - return cascadia.MustCompile(css) -} - -func lmatchfirst(node *html.Node, sel cascadia.Selector) *html.Node { - return sel.MatchFirst(node) -} - -func lmatchall(node *html.Node, sel cascadia.Selector) []*html.Node { - return sel.MatchAll(node) -} - -func lremovenode(node *html.Node) { - node.Parent.RemoveChild(node) -} - -func lwritestring(w io.Writer, s string) { - io.WriteString(w, s) -} - -func lreqgethost(req *http.Request) string { - return req.URL.Hostname() -} - -func lreqsethost(req *http.Request, host string) { - req.URL.Host = host -} - -func lreqgetpath(req *http.Request) string { - return req.URL.Path -} - -func lreqsetpath(req *http.Request, path string) { - req.URL.Path = path -} - -func lrespgetheader(resp *http.Response, name string) string { - return resp.Header.Get(name) -} - -func lclean(w io.Writer, node *html.Node, req *http.Request) { - clean(w, node, req.URL) -} - -func lsavehtml(filename string, node *html.Node) string { - fd, err := os.Create(filename) - if err != nil { - return err.Error() - } - defer fd.Close() - html.Render(fd, node) - return "ok" -} - -func addfunctions(L *lua.LState) { - L.SetGlobal("cssfilter", L.NewFunction(func2lua(lcssfilter))) - L.SetGlobal("matchfirst", L.NewFunction(func2lua(lmatchfirst))) - L.SetGlobal("matchall", L.NewFunction(func2lua(lmatchall))) - L.SetGlobal("removenode", L.NewFunction(func2lua(lremovenode))) - L.SetGlobal("clean", L.NewFunction(func2lua(lclean))) - L.SetGlobal("savehtml", L.NewFunction(func2lua(lsavehtml))) - L.SetGlobal("writestring", L.NewFunction(func2lua(lwritestring))) - L.SetGlobal("reqgethost", L.NewFunction(func2lua(lreqgethost))) - L.SetGlobal("reqsethost", L.NewFunction(func2lua(lreqsethost))) - L.SetGlobal("reqgetpath", L.NewFunction(func2lua(lreqgetpath))) - L.SetGlobal("reqsetpath", L.NewFunction(func2lua(lreqsetpath))) - L.SetGlobal("respgetheader", L.NewFunction(func2lua(lrespgetheader))) -} diff --git a/luainterface.go b/luainterface.go deleted file mode 100644 index 0825130..0000000 --- a/luainterface.go +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (c) 2017 Ted Unangst <tedu@tedunangst.com> - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -package main - -import ( - "log" - "reflect" - "sync" - - "github.com/yuin/gopher-lua" -) - -type Interpreter struct { - L *lua.LState -} - -var runqmtx sync.Mutex -var runnerqueue []*Interpreter - -func getinterpreter() *Interpreter { - runqmtx.Lock() - var runner *Interpreter - if len(runnerqueue) > 0 { - defer runqmtx.Unlock() - runner = runnerqueue[len(runnerqueue)-1] - runnerqueue = runnerqueue[0 : len(runnerqueue)-1] - } else { - runqmtx.Unlock() - L := lua.NewState() - addfunctions(L) - err := L.DoFile("filter.lua") - if err != nil { - log.Printf("error running file %s\n", err) - } - runner = &Interpreter{ - L: L, - } - } - return runner -} - -func putinterpreter(runner *Interpreter) { - alwaysreload := true - if alwaysreload { - runner.L.Close() - } else { - runqmtx.Lock() - defer runqmtx.Unlock() - runnerqueue = append(runnerqueue, runner) - } -} - -func func2lua(gofunc interface{}) lua.LGFunction { - return func(L *lua.LState) int { - rf := reflect.ValueOf(gofunc) - ft := reflect.TypeOf(gofunc) - numargs := ft.NumIn() - var args []reflect.Value - for i := 0; i < numargs; i++ { - var gv interface{} - lt := L.Get(i + 1).Type() - switch lt { - case lua.LTString: - gv = L.ToString(i + 1) - case lua.LTNumber: - gv = L.ToNumber(i + 1) - case lua.LTBool: - gv = L.ToBool(i + 1) - case lua.LTUserData: - gv = L.ToUserData(i + 1).Value - case lua.LTNil: - gv = reflect.Zero(ft.In(i)).Interface() - default: - panic("can't convert type from lua") - } - args = append(args, reflect.ValueOf(gv)) - } - rvs := rf.Call(args) - for i := 0; i < len(rvs); i++ { - pushsomething(L, rvs[i]) - } - return len(rvs) - } -} - -func (runner *Interpreter) Call(funcname string, args ...interface{}) string { - L := runner.L - L.Push(L.GetGlobal(funcname)) - for i := 0; i < len(args); i++ { - pushsomething(L, reflect.ValueOf(args[i])) - } - L.Call(len(args), 1) - rv := L.ToString(1) - L.Pop(1) - return rv -} - -func pushsomething(L *lua.LState, rv reflect.Value) { - switch rv.Kind() { - case reflect.Bool: - L.Push(lua.LBool(rv.Bool())) - case reflect.Int: - case reflect.Int8: - case reflect.Int16: - case reflect.Int32: - case reflect.Int64: - case reflect.Uint: - case reflect.Uint8: - case reflect.Uint16: - case reflect.Uint32: - case reflect.Uint64: - L.Push(lua.LNumber(rv.Int())) - case reflect.Float32: - case reflect.Float64: - L.Push(lua.LNumber(rv.Float())) - case reflect.String: - L.Push(lua.LString(rv.String())) - case reflect.Slice: - table := L.NewTable() - for i := 0; i < rv.Len(); i++ { - appendsomething(L, table, rv.Index(i)) - } - L.Push(table) - case reflect.Ptr: - if rv.IsNil() { - L.Push(lua.LNil) - } else { - ud := L.NewUserData() - ud.Value = rv.Interface() - L.Push(ud) - } - default: - ud := L.NewUserData() - ud.Value = rv.Interface() - L.Push(ud) - } -} - -func appendsomething(L *lua.LState, table *lua.LTable, rv reflect.Value) { - switch rv.Kind() { - case reflect.Bool: - table.Append(lua.LBool(rv.Bool())) - case reflect.Int: - case reflect.Int8: - case reflect.Int16: - case reflect.Int32: - case reflect.Int64: - case reflect.Uint: - case reflect.Uint8: - case reflect.Uint16: - case reflect.Uint32: - case reflect.Uint64: - table.Append(lua.LNumber(rv.Int())) - case reflect.Float32: - case reflect.Float64: - table.Append(lua.LNumber(rv.Float())) - case reflect.String: - table.Append(lua.LString(rv.String())) - case reflect.Slice: - innertable := L.NewTable() - for i := 0; i < rv.Len(); i++ { - appendsomething(L, innertable, rv.Index(i)) - } - table.Append(innertable) - default: - ud := L.NewUserData() - ud.Value = rv.Interface() - table.Append(ud) - } -} diff --git a/miniwebproxy.go b/miniwebproxy.go index 11ba199..61c95de 100644 --- a/miniwebproxy.go +++ b/miniwebproxy.go @@ -69,9 +69,8 @@ func getDialer() contextDialer { return &net.Dialer{} } -func finishinterpreter(L *Interpreter) { - defer putinterpreter(L) - L.Call("endconnection") +func finishinterpreter(interp *interpreter) { + defer putinterpreter(interp) } // this should not be different from the TLS intercept proxy @@ -110,14 +109,14 @@ func proxyreq(w http.ResponseWriter, r *http.Request) { return } - L := getinterpreter() - defer finishinterpreter(L) - L.Call("newconnection", r) - if !shouldintercept(L, r.URL.Hostname()) { + interp := getinterpreter() + defer finishinterpreter(interp) + interp.NewConnection(r) + if !interp.ShouldIntercept(r.URL.Hostname()) { resp.Write(clientconn) return } - filter(L, clientconn, resp, r) + filter(interp, clientconn, resp, r) } type expiringConfig struct { @@ -269,10 +268,10 @@ func (pxr *Proxer) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - L := getinterpreter() - defer finishinterpreter(L) - L.Call("newconnection", r) - if !shouldintercept(L, desthost) { + interp := getinterpreter() + defer finishinterpreter(interp) + interp.NewConnection(r) + if !interp.ShouldIntercept(desthost) { serverconn, err := connect(ctx, dest, "") if err != nil { log.Printf("failed to connect: %s", err) @@ -325,7 +324,7 @@ func (pxr *Proxer) ServeHTTP(w http.ResponseWriter, r *http.Request) { clientreq.Header.Set("Accept-Encoding", "gzip") } log.Printf("clientreq url %s/%s\n", clientreq.URL.Hostname(), clientreq.URL.Path) - prefilter(L, clientreq) + interp.Prefilter(clientreq) clientreq.URL.Host = clientreq.URL.Hostname() if clientreq.Header.Get("Upgrade") == "websocket" { @@ -351,7 +350,7 @@ func (pxr *Proxer) ServeHTTP(w http.ResponseWriter, r *http.Request) { writeerror(clientconn, "error proxying request") return } - filter(L, clientconn, serverresp, clientreq) + filter(interp, clientconn, serverresp, clientreq) log.Printf("finished %s%s", clientreq.URL.Hostname(), clientreq.URL.Path) } } |