Support parsing rsync://

# rsync2os

## rsync the remote files to your object storage

## Sync remote files from rsync server to your object storage

![client](https://github.com/kaikaz/rsync2os/blob/master/docs/client.jpg)

![client](https://raw.githubusercontent.com/kaiakz/rsync2os/master/docs/client.jpg)

# HandShake

## Why we don't need block checksum?

Rsync requires random reading and writing of files to do the block exchange. But object storage does not support that.

Rsync2os simplifies the rsync algorithm to avoid random reading and writing. When a file needs to be updated, we just download the entire file from the server and then replace it.

## HandShake

rysnc2os uses rsync protocol 27. It sends the arguments "--server--sender-l-p-r-t" to the remote rsyncd.

# The File List

## The File List

According the arguments rsync2os sent, the file list should contain path, size, modified time & mode.

# Request the file

## Request the file

rsync2os always saves the file list in its database(local file list). rsync2os doesn't compare each file with the file list from remote server(remote file list), but the latest local file list. If the file in the local file list has different size, modified time, or doesn't exist, rsync2os will download the whole file(without [block exchange](https://github.com/kristapsdz/openrsync#block-exchange)). To to do that, rsync2os sends the empty block checksum with the file's index in the remote file list. 

# Download the file

## Download the file

The rsync server sends the entire file as a stream of bytes.

# Multiplex & De-Multiplex

## Multiplex & De-Multiplex

# How to use the demo?

## How to use the demo?

1. install & run minio, you can configure in WriteOS

2. go run main.go

# Reference

* https://git.samba.org/?p=rsync.git

* https://rsync.samba.org/resources.html

* https://github.com/openbsd/src/tree/master/usr.bin/rsync

* https://github.com/tuna/rsync

* https://github.com/sourcefrog/rsyn

* https://github.com/gilbertchen/acrosync-library

* https://github.com/boundary/wireshark/blob/master/epan/dissectors/packet-rsync.c

* https://tools.ietf.org/html/rfc5781

 No newline at end of file

package main

import (

	"fmt"

	"io"

	"log"

	"net"

func Client() {

	conn, err := net.Dial("tcp", "101.6.8.193:873")

	// tuna: mirrors.tuna.tsinghua.edu.cn 101.6.8.193:873

	if err != nil {

		// TODO

	}

	data := make(chan byte, 16 * 1024 * 1024)

	// Start De-Multiplexing

	go rsync.DeMuxChan(conn, data)

	filelist := make(rsync.FileList, 0, 3072)

	filelist := make(rsync.FileList, 0, 4096)

	// recv_file_list

	for {

		if rsync.GetFileList(data, &filelist) == io.EOF {

}

func main() {

	Client()

	//Client()

	fmt.Println(rsync.SplitURI("rsync://mirrors.tuna.tsinghua.edu.cn:1080/elvish"))

	fmt.Println(rsync.SplitURI("rsync://mirror.tuna.tsinghua.edu.cn:1080000/epel/7/SRPMS"))

}

	// receive server's protocol version and seed

	version_str, _ := ReadLine(conn)

	// recv(version)

	var remote_protocol, remote_sub int

	fmt.Sscanf(version_str, "@RSYNCD: %d.%d", remote_protocol, remote_sub)

	log.Println(version_str)

	// recv(version)

	// scanf(version, "@RSYNCD: %d.%d", )

	// send mod name

	// send("Foo\n")

	conn.Write([]byte("epel\n"))

}

func Generate(conn net.Conn, filelist *FileList) {

	// Compare all local files with file list, pick up the files that has different size, mtime

	// Those files are `basis files`

	"encoding/binary"

	"fmt"

	"net"

	"strconv"

	"strings"

)

func ReadLine(conn net.Conn) (string, error) {

	return int64(binary.LittleEndian.Uint64(data))

}

func ReadFList(conn net.Conn) {

func ReadExact(conn net.Conn, b []byte) (int, error) {

	for i:= 0; i < len(b); {

		n, err := conn.Read(b[i:])

		if err != nil {

			return n, nil

		}

		i += n

	}

	return len(b), nil

}

	flags := ReadByte(conn)

// For rsync

func SplitURIS(uri string) (string, int, string, string, error){

	var partial, pathlen uint32 = 0, 0

	var host, module, path string

	var first = []byte(uri)

	var second []byte

	fmt.Println(flags)

	/*

	 * Read our filename.

	 * If we have FLIST_NAME_SAME, we inherit some of the last

	 * transmitted name.

	 * If we have FLIST_NAME_LONG, then the string length is greater

	 * than byte-size.

	 */

	if (0x20 & flags) != 0 {

		partial = uint32(ReadByte(conn))

		fmt.Println("Partical", partial)

	if strings.HasPrefix(uri, "rsync://") {

		/* rsync://host[:port]/module[/path] */

		first = first[8:]

		i := bytes.IndexByte(first, '/')

		if i == -1 {

			// No module name

			panic("No module name")

		}

	/* Get the (possibly-remaining) filename length. */

	if (0x40 & flags) != 0 {

		pathlen = uint32(ReadInteger(conn)) // can't use for rsync 31

		// Var int

		// i := readByte(conn)

		// var j byte, len int

		// for j = 0; j<=6; ++j {

		// 	if ((i & 0x80) == 0)	break

		// 	c =

		// }

		second = first[i+1:]	//ignore '/'

		first = first[:i]

	} else {

		pathlen = uint32(ReadByte(conn))

		// Only for remote

		/* host::module[/path] */

		panic("No implement yet")

	}

	fmt.Println("PathLen", pathlen)

	/* Allocate our full filename length. */

	/* FIXME: maximum pathname length. */

	// if pathlen + partical == 0

	// malloc len error?

	//

	if (0x20 & flags) != 0 {

		// return last 4096bytes

	port := 873		// Default port: 873

	// Parse port

	i := bytes.IndexByte(first, ':')

	if i != -1  {

		var err error

		port, err = strconv.Atoi(string(first[i+1:]))

		if err != nil {

			// Wrong port

			panic("Wrong port")

		}

		first = first[:i]

	}

	host = string(first)

	path, _ := ReadBuffer(conn, pathlen)

	fmt.Println("Path", path)

	// Parse path

	i = bytes.IndexByte(second, '/')

	if i != -1 {

		path = string(second[i:])

		second = second[:i]

	}

	module = string(second)

	size := ReadInteger(conn)

	fmt.Print("Size", size)

	return host, port, module, path, nil

	if (flags & 0x80) == 0 {

		fmt.Println("MTIME", ReadInteger(conn))

}

	var mode int32

	if (flags & 0x02) == 0 {

		mode = ReadInteger(conn)

		fmt.Println("Mode", mode)

	}

// For rsync

func SplitURI(uri string) (string, string, string, error){

	if ((mode & 32768) != 0) && ((mode & 8192) != 0) {

		len := uint32(ReadInteger(conn))

		slink, _ := ReadBuffer(conn, len)

		fmt.Println("Symbolic", len, "is", slink)

	}

	var address, module, path string

	var first = []byte(uri)

	var second []byte

	//return "\n"

	if strings.HasPrefix(uri, "rsync://") {

		/* rsync://host[:port]/module[/path] */

		first = first[8:]

		i := bytes.IndexByte(first, '/')

		if i == -1 {

			// No module name

			panic("No module name")

		}

		second = first[i+1:]	//ignore '/'

		first = first[:i]

	} else {

		// Only for remote

		/* host::module[/path] */

		panic("No implement yet")

	}

func ReadExact(conn net.Conn, b []byte) (int, error) {

	for i:= 0; i < len(b); {

		n, err := conn.Read(b[i:])

		if err != nil {

			return n, nil

	address = string(first)

	// Parse port

	i := bytes.IndexByte(first, ':')

	if i == -1  {

		address += ":873"	// Default port: 873

	}

		i += n

	// Parse path

	i = bytes.IndexByte(second, '/')

	if i != -1 {

		path = string(second[i:])

		second = second[:i]

	}

	return len(b), nil

	module = string(second)

	return address, module, path, nil

}

 No newline at end of file