Supports full downloading (a2704277) · Commits · SUMMER2020 / students / proj-2027121

main.go

+35 −3

Original line number	Diff line number	Diff line
		@@ -10,6 +10,7 @@ package main

		import (
		"fmt"
		"github.com/minio/minio-go/v6"
		"io"
		"log"
		"net"
		@@ -37,6 +38,7 @@ func Client(uri string) {

		if err != nil {
		// TODO
		panic("Network Error")
		}

		defer conn.Close()
		@@ -67,12 +69,38 @@ func Client(uri string) {
		// Sort the filelist lexicographically
		sort.Sort(filelist)



		// Init the object storage
		// For test
		endpoint := "127.0.0.1:9000"
		accessKeyID := "minioadmin"
		secretAccessKey := "minioadmin"

		minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, false)
		if err != nil {
		panic("Failed")
		}

		// Create a bucket for the module
		err = minioClient.MakeBucket(module, "us-east-1")
		if err != nil {
		// Check to see if we already own this bucket (which happens if you run this twice)
		exists, errBucketExists := minioClient.BucketExists(module)
		if errBucketExists == nil && exists {
		log.Printf("We already own %s\n", module)
		} else {
		log.Fatalln(err)
		}
		} else {
		log.Printf("Successfully created %s\n", module)
		}

		// Generate target file list
		//rsync.RequestAFile(conn, "libnemo-extension1_1.8.1+maya_amd64.deb", &filelist)
		//rsync.GetFiles(data, conn, &filelist)


		rsync.RequestFiles(conn, data, &filelist)
		rsync.RequestFiles(conn, data, &filelist, minioClient, module, path)
		//go rsync.Downloader(data, &filelist)
		//fmt.Println(filelist)

		@@ -82,7 +110,11 @@ func Client(uri string) {
		}

		func main() {
		Client("rsync://mirrors.kernel.org/linuxmint-packages/pool/romeo/n/nemo/")
		//FIXME: Can't handle wrong module/path rsync://mirrors.tuna.tsinghua.edu.cn/linuxmint-packages/pool/romeo/libf/libfm/
		Client("rsync://mirrors.tuna.tsinghua.edu.cn/elvish")
		//Client("rsync://rsync.monitoring-plugins.org/plugins/")
		//Client("rsync://rsync.mirrors.ustc.edu.cn/repo/monitoring-plugins")
		// rsync://rsync.monitoring-plugins.org/plugins/
		}

rsync/demultiplex.go

+2 −1

Original line number	Diff line number	Diff line
		@@ -222,6 +222,7 @@ func lookup(size int64, filelist *FileList) {
		}
		}

		// Test
		func GetFile(data chan byte, index int32, filelist *FileList) {

		path := (*filelist)[index].Path

rsync/receiver.go

+15 −18

Original line number	Diff line number	Diff line
		@@ -4,6 +4,7 @@ import (
		"bytes"
		"encoding/binary"
		"fmt"
		"github.com/minio/minio-go/v6"
		"io"
		"log"
		"net"
		@@ -182,11 +183,8 @@ func GetFileList(data chan byte, filelist *FileList) error {

		/* Generator */

		func RequestFiles(conn net.Conn, data chan byte, filelist *FileList) {
		func RequestFiles(conn net.Conn, data chan byte, filelist FileList, os minio.Client, module string, ppath string) {
		empty := make([]byte, 16) // 4 + 4 + 4 + 4 bytes
		//downloading := false


		for i:=0; i < len(*filelist); i++ {
		if (*filelist)[i].Mode == 0100644 {
		binary.Write(conn, binary.LittleEndian, int32(i))
		@@ -194,21 +192,13 @@ func RequestFiles(conn net.Conn, data chan byte, filelist *FileList) {
		fmt.Println((*filelist)[i].Path)
		conn.Write(empty)

		//ni := GetInteger(data)
		//fmt.Println(ni)
		//GetFile(data, int32(ni), filelist)

		//if !downloading {
		// downloading = false
		// go Downloader(data, filelist)
		//}
		}

		}
		fmt.Println("FINISH")
		log.Println("Request completed")
		// Finish
		binary.Write(conn, binary.LittleEndian, int32(-1))
		Downloader(data, filelist)
		Downloader(data, filelist, os, module, ppath)
		}

		func RequestAFile(conn net.Conn, target string, filelist *FileList) {
		@@ -242,12 +232,10 @@ func RequestAFile(conn net.Conn, target string, filelist *FileList) {
		// Empty checksum

		// Finish
		//binary.Write(conn, binary.LittleEndian, int32(-1))

		binary.Write(conn, binary.LittleEndian, int32(-1))
		}

		// Goroutine
		func Downloader(data chan byte, filelist *FileList) {
		func Downloader(data chan byte, filelist FileList, os minio.Client, module string, ppath string) {
		for {
		index := GetInteger(data)
		if index == -1 {
		@@ -277,6 +265,15 @@ func Downloader(data chan byte, filelist *FileList) {
		buf.Write(ctx)
		}
		}

		// Put file to object storage
		n, err := os.PutObject(module, (ppath+path)[1:], buf, int64(buf.Len()), minio.PutObjectOptions{})
		if err != nil {
		log.Fatalln(err)
		}

		log.Printf("Successfully uploaded %s of size %d\n", path, n)

		// Remote MD4
		rmd4 := make([]byte, 16)
		GetBytes(data, rmd4)