fix: #64 OOM

cmd/ipasd/ipasd.go

@@ -13,9 +13,11 @@ import (
 
 	httptransport "github.com/go-kit/kit/transport/http"
 	"github.com/iineva/ipa-server/cmd/ipasd/service"
+	"github.com/iineva/ipa-server/pkg/common"
 	"github.com/iineva/ipa-server/pkg/httpfs"
 	"github.com/iineva/ipa-server/pkg/storager"
 	"github.com/iineva/ipa-server/pkg/uuid"
+	"github.com/iineva/ipa-server/pkg/websocketfile"
 	"github.com/iineva/ipa-server/public"
 )
 
@@ -136,6 +138,42 @@ func main() {
 	serve.Handle("/api/delete", deleteHandler)
 	serve.Handle("/api/delete/get", deleteGetHandler)
 	serve.Handle("/plist/", plistHandler)
+	// upload file over Websocket
+	serve.Handle("/api/upload/ws", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		f, err := websocketfile.NewWebsocketFile(w, r)
+		if err != nil {
+			logger.Log("msg", fmt.Sprintf("err: %v", err))
+			return
+		}
+
+		size, err := f.Size()
+		if err != nil {
+			logger.Log("msg", fmt.Sprintf("err: %v", err))
+			return
+		}
+
+		name, err := f.Name()
+		if err != nil {
+			logger.Log("msg", fmt.Sprintf("err: %v", err))
+			return
+		}
+		t := service.FileType(name)
+
+		logger.Log("name:", name, " size:", size)
+
+		info, err := srv.Add(f, size, t)
+		if err != nil {
+			logger.Log("msg", fmt.Sprintf("err: %v", err))
+			return
+		}
+
+		err = f.Done(common.ToMap(info))
+		if err != nil {
+			logger.Log("msg", fmt.Sprintf("err: %v", err))
+			return
+		}
+
+	}))
 
 	// static files
 	uploadFS := afero.NewBasePathFs(afero.NewOsFs(), *storageDir)

cmd/ipasd/service/service.go

@@ -69,14 +69,13 @@ type Service interface {
 	Find(id string, publicURL string) (*Item, error)
 	History(id string, publicURL string) ([]*Item, error)
 	Delete(id string) error
-	Add(r Reader, t AppInfoType) (*AppInfo, error)
+	Add(r Reader, size int64, t AppInfoType) (*AppInfo, error)
 	Plist(id, publicURL string) ([]byte, error)
 }
 
 type Reader interface {
 	io.Reader
 	io.ReaderAt
-	Size() int64
 }
 
 type service struct {
@@ -176,9 +175,9 @@ func (s *service) Delete(id string) error {
 	return nil
 }
 
-func (s *service) Add(r Reader, t AppInfoType) (*AppInfo, error) {
+func (s *service) Add(r Reader, size int64, t AppInfoType) (*AppInfo, error) {
 
-	app, err := s.addPackage(r, t)
+	app, err := s.addPackage(r, size, t)
 	if err != nil {
 		return nil, err
 	}
@@ -191,7 +190,7 @@ func (s *service) Add(r Reader, t AppInfoType) (*AppInfo, error) {
 	return app, s.saveMetadata()
 }
 
-func (s *service) addPackage(r Reader, t AppInfoType) (*AppInfo, error) {
+func (s *service) addPackage(r Reader, size int64, t AppInfoType) (*AppInfo, error) {
 	// save ipa file to temp
 	pkgTempFileName := filepath.Join(tempDir, uuid.NewString())
 	if err := s.store.Save(pkgTempFileName, r); err != nil {
@@ -203,9 +202,9 @@ func (s *service) addPackage(r Reader, t AppInfoType) (*AppInfo, error) {
 	var err error
 	switch t {
 	case AppInfoTypeIpa:
-		pkg, err = ipa.Parse(r, r.Size())
+		pkg, err = ipa.Parse(r, size)
 	case AppInfoTypeApk:
-		pkg, err = apk.Parse(r, r.Size())
+		pkg, err = apk.Parse(r, size)
 	}
 	if err != nil {
 		return nil, err

cmd/ipasd/service/transport.go

@@ -73,7 +73,7 @@ func MakeAddEndpoint(srv Service) endpoint.Endpoint {
 			return nil, fmt.Errorf("do not support %s file", path.Ext(p.file.FileName()))
 		}
 
-		app, err := srv.Add(buf, t)
+		app, err := srv.Add(buf, buf.Size(), t)
 		if err != nil {
 			return nil, err
 		}

go.mod

@@ -11,10 +11,10 @@ require (
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/go-kit/kit v0.10.0
 	github.com/google/uuid v1.2.0 // indirect
-	// TODO: wait PR merge github.com/poolqa/CgbiPngFix master
-	github.com/iineva/CgbiPngFix v0.0.0-20210523041253-b8869b346914
+	github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c
 	github.com/iineva/bom v0.0.0-20210604102127-81d8bcf0765e
 	github.com/lithammer/shortuuid v3.0.0+incompatible
+	github.com/poolqa/CgbiPngFix v0.0.0-20211024081647-8ad4fb5c23e4
 	github.com/qiniu/go-sdk/v7 v7.9.5
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/shogo82148/androidbinary v1.0.2

go.sum

@@ -120,6 +120,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -146,8 +147,6 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
-github.com/iineva/CgbiPngFix v0.0.0-20210523041253-b8869b346914 h1:l/3SkoXjUmXjj90Ho9FHksi6OVJDIOGU0Ow2KSZnNjE=
-github.com/iineva/CgbiPngFix v0.0.0-20210523041253-b8869b346914/go.mod h1:caygohjd8B7TXErEL2NYMdJNLO9jIddQIALUKcoszZo=
 github.com/iineva/bom v0.0.0-20210604102127-81d8bcf0765e h1:LRrJgM6YtcVTzuDviS+xIglYVe/AbpqGCHD2bDtFI8Y=
 github.com/iineva/bom v0.0.0-20210604102127-81d8bcf0765e/go.mod h1:O7ivqgPZeHUc3Dk9ZA+bqH/+7Qr/2lo1pogmk78RaQQ=
 github.com/iineva/go-lzfse v1.1.13-0.20210604101847-2a555776c20a h1:c8iYO/eYhs1ec6n4oLzl7x0qt6o2ncMo5Tu4RqaTjNY=
@@ -231,6 +230,8 @@ github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6J
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/poolqa/CgbiPngFix v0.0.0-20211024081647-8ad4fb5c23e4 h1:n3IkiWQgZS8Ref8BX/NWpYe7VClpQJ8pbzUSBZOqoKs=
+github.com/poolqa/CgbiPngFix v0.0.0-20211024081647-8ad4fb5c23e4/go.mod h1:DG0Kw4br1oA1IRv7O/QNp8fzh+/YBgknQCLnMeioG1U=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=

pkg/common/common.go

@@ -1,5 +1,7 @@
 package common
 
+import "encoding/json"
+
 // get args until arg is not empty
 func Def(args ...string) string {
 	for _, v := range args {
@@ -9,3 +11,14 @@ func Def(args ...string) string {
 	}
 	return ""
 }
+
+// 结构体转 map
+func ToMap(v interface{}) map[string]interface{} {
+	b, err := json.Marshal(v)
+	m := map[string]interface{}{}
+	if err != nil {
+		return m
+	}
+	_ = json.Unmarshal(b, &m)
+	return m
+}

pkg/ipa/ipa.go

@@ -11,11 +11,10 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/iineva/CgbiPngFix/ipaPng"
-
 	"github.com/iineva/bom/pkg/asset"
 	"github.com/iineva/ipa-server/pkg/plist"
 	"github.com/iineva/ipa-server/pkg/seekbuf"
+	"github.com/poolqa/CgbiPngFix/ipaPng"
 )
 
 var (

pkg/storager/afero.go

@@ -1,6 +1,7 @@
 package storager
 
 import (
+	"bufio"
 	"io"
 	"path/filepath"
 
@@ -14,6 +15,7 @@ type oferoStorager struct {
 
 const (
 	oferoStoragerDirPerm = 0755
+	WRITER_BUFFER_SIZE   = 1024 * 1024 * 2 // 1M
 )
 
 var _ Storager = (*oferoStorager)(nil)
@@ -36,10 +38,25 @@ func (f *oferoStorager) Save(name string, reader io.Reader) error {
 		return err
 	}
 	fi, err := f.fs.Create(name)
+	defer func() {
+		_ = fi.Close()
+	}()
 	if err != nil {
 		return err
 	}
-	_, err = io.Copy(fi, reader)
+
+	// write with buffer
+	w := bufio.NewWriterSize(fi, WRITER_BUFFER_SIZE)
+	_, err = io.Copy(w, reader)
+	if err != nil {
+		return err
+	}
+
+	err = w.Flush()
+	if err != nil {
+		return err
+	}
+
 	return err
 }