Test filelist cache (2afa3f88) · Commits · SUMMER2020 / students / proj-2027121

filelist/cache.go

+72 −14

Original line number	Diff line number	Diff line
		package filelist

		import (
		"google.golang.org/protobuf/proto"
		"fmt"
		"log"
		"rsync2os/rsync"

		bolt "go.etcd.io/bbolt"
		"google.golang.org/protobuf/proto"
		)

		type Cache struct {
		@@ -12,20 +15,26 @@ type Cache struct {
		prepath string
		}

		func Open(module string, prepath string) {
		func Open(module string, prepath string) *Cache {
		db, err := bolt.Open("r.db", 0666, nil)
		if err != nil {
		return
		return nil
		}
		tx, err := db.Begin(true)
		if tx == nil {
		return
		return nil
		}
		defer tx.Rollback()

		_, err = tx.CreateBucketIfNotExists([]byte(module))
		if err := tx.Commit(); err != nil {

		var bucket *bolt.Bucket
		bucket, err = tx.CreateBucketIfNotExists([]byte(module))
		if err != nil {
		return nil
		}
		return &Cache{
		db: db,
		module: bucket,
		prepath: prepath,
		}
		}

		@@ -56,14 +65,63 @@ func (cache Cache) Get(key []byte) FInfo {

		func (cache Cache) PutAll(list rsync.FileList) error {
		for _, info := range *list {
		err := cache.Put(&info);
		err := cache.Put(&info)
		if err != nil {
		return err
		}
		}
		return nil
		}

		// Test
		func Save(list *rsync.FileList, module string, prepath string) {
		db, err := bolt.Open("test.db", 0666, nil)
		if err != nil {
		return
		}

		err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(module))
		if err != nil {
		log.Panicln("create module as bucket failed", err)
		return err
		}
		for _, info := range *list {
		key := []byte(prepath + info.Path)
		value, err := proto.Marshal(&FInfo{
		Size: info.Size,
		Mtime: info.Mtime,
		Mode: info.Mode,
		})
		if err != nil {
		log.Println("Marshal failed", err)
		return err
		}
		bucket.Put(key, value)
		}
		return nil
		})
		if err != nil {
		log.Println("Update failed", err)
		}

		err = db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket([]byte(module))

		c := b.Cursor()

		for k, v := c.First(); k != nil; k, v = c.Next() {
		fmt.Println("key= ", string(k))
		var m FInfo
		proto.Unmarshal(v, &m)
		fmt.Println(m)
		}

		func Save(list *rsync.FileList) {
		return nil
		})

		if err != nil {

		}
		}

go.mod

+3 −2

Original line number	Diff line number	Diff line
		@@ -3,8 +3,9 @@ module rsync2os
		go 1.14

		require (
		github.com/golang/protobuf v1.4.1
		github.com/minio/minio-go/v6 v6.0.56
		go.etcd.io/bbolt v1.3.4 // indirect
		go.etcd.io/bbolt v1.3.4
		golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37
		google.golang.org/protobuf v1.24.0 // indirect
		google.golang.org/protobuf v1.24.0
		)

go.sum

+1 −0

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
		github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
		github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
		github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
		github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
		github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
		github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
		github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=

main.go

+13 −13

Original line number	Diff line number	Diff line
		@@ -10,14 +10,15 @@ package main

		import (
		"fmt"
		"github.com/minio/minio-go/v6"
		"io"
		"log"
		"net"
		"rsync2os/filelist"
		"rsync2os/rsync"
		"sort"
		)

		"github.com/minio/minio-go/v6"
		)

		func Client(uri string) {

		@@ -42,26 +43,29 @@ func Client(uri string) {

		data := make(chan byte, 1610241024)


		// Start De-Multiplexing
		go rsync.DeMuxChan(conn, data)

		filelist := make(rsync.FileList, 0, 4096)
		filelist1 := make(rsync.FileList, 0, 4096)
		// recv_file_list
		for {
		if rsync.GetFileList(data, &filelist) == io.EOF {
		if rsync.GetFileList(data, &filelist1) == io.EOF {
		break
		}
		}
		log.Println("File List Received, total size is", len(filelist))
		log.Println("File List Received, total size is", len(filelist1))

		ioerr := rsync.GetInteger(data)
		log.Println("IOERR", ioerr)

		// Sort the filelist lexicographically
		sort.Sort(filelist)
		sort.Sort(filelist1)

		ppath := rsync.TrimPrepath(path)
		filelist.Save(&filelist1, module, ppath)
		log.Println("File List Saved")

		return

		// Init the object storage
		// For test
		@@ -92,12 +96,10 @@ func Client(uri string) {
		//rsync.RequestAFile(conn, "libnemo-extension1_1.8.1+maya_amd64.deb", &filelist)
		//rsync.GetFiles(data, conn, &filelist)

		rsync.RequestFiles(conn, data, &filelist, minioClient, module, path)
		// rsync.RequestFiles(conn, data, &filelist, minioClient, module, path)
		//go rsync.Downloader(data, &filelist)
		//fmt.Println(filelist)



		}

		func main() {
		@@ -107,5 +109,3 @@ func main() {
		//Client("rsync://rsync.mirrors.ustc.edu.cn/repo/monitoring-plugins")
		// rsync://rsync.monitoring-plugins.org/plugins/
		}

rsync/demultiplex.go

+55 −55

Original line number	Diff line number	Diff line
		@@ -3,9 +3,11 @@ package rsync
		import (
		"bytes"
		"encoding/binary"
		"io"

		"github.com/minio/minio-go/v6"
		"golang.org/x/crypto/md4"
		"io"

		//"io/ioutil"
		"log"
		"net"
		@@ -68,8 +70,6 @@ func DeMuxBuf(conn net.Conn, buf *bytes.Buffer) {
		return
		}



		} else { // out-of-band data
		//otag := tag - 7

		@@ -108,8 +108,6 @@ func DeMultiplex(conn net.Conn) error {
		// return io.EOF
		//}



		//for _, b := range body {
		// data <- b
		//}
		@@ -130,7 +128,9 @@ func DeMuxChan(conn net.Conn, data chan byte) {

		n, err := ReadExact(conn, header)
		if n != 4 \|\| err != nil {
		panic("Mulitplex: Check your wired protocol")
		// panic("Mulitplex: Check your wired protocol")
		log.Println("Mulitplex: Check your wired protocol")
		return
		}

		tag := header[3] // Little Endian
		@@ -215,7 +215,7 @@ func GetFiles(data chan byte, conn net.Conn, filelist *FileList) {
		}

		func lookup(size int64, filelist *FileList) {
		for i,f := range(*filelist) {
		for i, f := range *filelist {
		if f.Size == size {
		log.Println("True File:", i, f)
		}