Commit dd19fdd3 authored by 黄大凯's avatar 黄大凯
Browse files

Rename rsync2os to rsync-os

parent c4b2fe9a

README.md

+9 −7
Original line number Diff line number Diff line 
# rsync2os

## Sync remote files from rsync server to your object storage

# RSYNC-OS

## A rsync gateway for object storage.



![client](https://raw.githubusercontent.com/kaiakz/rsync2os/master/docs/client.jpg)



## Why we don't need block checksum?

Rsync requires random reading and writing of files to do the block exchange. But object storage does not support that.

Rsync2os simplifies the rsync algorithm to avoid random reading and writing. When a file needs to be updated, we just download the entire file from the server and then replace it.

rsync-os simplifies the rsync algorithm to avoid random reading and writing. When a file needs to be updated, we just download the entire file from the server and then replace it.



## HandShake

rysnc2os uses rsync protocol 27. It sends the arguments "--server--sender-l-p-r-t" to the remote rsyncd.

rysnc-os uses rsync protocol 27. 

It sends the arguments "--server--sender-l-p-r-t" to the remote rsyncd by default.



## The File List

According the arguments rsync2os sent, the file list should contain path, size, modified time & mode.

According the arguments rsync-os sent, the file list should contain path, size, modified time & mode. 

 

## Request the file

rsync2os always saves the file list in its database(local file list). rsync2os doesn't compare each file with the file list from remote server(remote file list), but the latest local file list. If the file in the local file list has different size, modified time, or doesn't exist, rsync2os will download the whole file(without [block exchange](https://github.com/kristapsdz/openrsync#block-exchange)). To to do that, rsync2os sends the empty block checksum with the file's index in the remote file list. 

rsync-os always saves the file list in its database(local file list). rsync2os doesn't compare each file with the file list from remote server(remote file list), but the latest local file list. If the file in the local file list has different size, modified time, or doesn't exist, rsync2os will download the whole file(without [block exchange](https://github.com/kristapsdz/openrsync#block-exchange)). To to do that, rsync2os sends the empty block checksum with the file's index of the remote file list. 



## Download the file

The rsync server sends the entire file as a stream of bytes.
@@ -23,7 +24,8 @@ The rsync server sends the entire file as a stream of bytes. 
![de-multiplex](https://raw.githubusercontent.com/kaiakz/rsync2os/master/docs/demux.jpg)



## How to use the demo?

1. install & run minio, you can configure in WriteOS() or Client()

### Use minio

1. install & run minio, you need to configure your setting of minio in the main.go.

2. go run main.go



# Reference

fldb/cache.go

+1 −55
Original line number Diff line number Diff line 
package fldb



import (

	"fmt"

	"log"

	"rsync2os/rsync"



	"rsync-os/rsync"

	bolt "go.etcd.io/bbolt"

	"google.golang.org/protobuf/proto"

)
@@ -73,55 +70,4 @@ func (cache *Cache) PutAll(list *rsync.FileList) error { 
	return nil

}



// Test

func Save(list *rsync.FileList, module string, prepath string) {

	db, err := bolt.Open("test.db", 0666, nil)

	if err != nil {

		return

	}



	err = db.Update(func(tx *bolt.Tx) error {

		bucket, err := tx.CreateBucketIfNotExists([]byte(module))

		if err != nil {

			log.Panicln("create module as bucket failed", err)

			return err

		}

		for _, info := range *list {

			key := []byte(prepath + info.Path)

			value, err := proto.Marshal(&FInfo{

				Size:  info.Size,

				Mtime: info.Mtime,

				Mode:  info.Mode,

			})

			if err != nil {

				log.Println("Marshal failed", err)

				return err

			}

			bucket.Put(key, value)

		}

		return nil

	})

	if err != nil {

		log.Println("Update failed", err)

	}



	err = db.View(func(tx *bolt.Tx) error {

		// Assume bucket exists and has keys

		b := tx.Bucket([]byte(module))



		c := b.Cursor()



		for k, v := c.First(); k != nil; k, v = c.Next() {

			fmt.Println("key= ", string(k))

			var m FInfo

			proto.Unmarshal(v, &m)

			fmt.Println(m.GetMtime(), m.GetSize())

		}



		return nil

	})



	if err != nil {

	}

}

fldb/diff.go

+1 −1
Original line number Diff line number Diff line 
package fldb



import (

	"rsync2os/rsync"

	"rsync-os/rsync"

)



// Diff two sorted list

fldb/finfo.pb.go

+1 −1
Original line number Diff line number Diff line
@@ -98,7 +98,7 @@ var file_finfo_proto_rawDesc = []byte{ 
	0x52, 0x05, 0x4d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x18,

	0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x42, 0x21, 0x5a, 0x1f, 0x67,

	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x61, 0x69, 0x61, 0x6b, 0x7a,

	0x2f, 0x72, 0x73, 0x79, 0x6e, 0x63, 0x32, 0x6f, 0x73, 0x2f, 0x66, 0x6c, 0x64, 0x62, 0x62, 0x06,

	0x2f, 0x72, 0x73, 0x79, 0x6e, 0x63, 0x2d, 0x6f, 0x73, 0x2f, 0x66, 0x6c, 0x64, 0x62, 0x62, 0x06,

	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,

}

fldb/finfo.proto

+1 −1
Original line number Diff line number Diff line 
syntax = "proto3";

package proto;



option go_package = "github.com/kaiakz/rsync2os/fldb";

option go_package = "github.com/kaiakz/rsync-os/fldb";



message FInfo {

    int64 Size = 1;