rsync
Description
rsync is an open source utility that provides fast incremental file transfer. rsync is freely available under the GNU General Public License and is currently being maintained by Wayne Davison.
Preparation for Installation
Start PuTTY on a Windows PC, Terminal on a Mac or similar terminal application on a Linux PC.
In this example Terminal on a Mac is used.
Open a remote SSH session to the server with:
Mac:~ user$ ssh user@192.168.1.4 [enter][user@server ~]$Enable superuser privileges with:
[user@server ~]$ sudo -s [enter]
Password: <-- passwd [enter]
[root@server /usr/home/user]#N.B.: Enter user password, not the root password!
Installation
Install the rsync port with:
[root@server /usr/home/user]# pkg install net/rsync [enter]
Updating FreeBSD repository catalogue...
FreeBSD repository is up to date.
All repositories are up to date.
The following 1 package(s) will be affected (of 0 checked):
New packages to be INSTALLED:
rsync: 3.1.2_7
Number of packages to be installed: 1
298 KiB to be downloaded.
Proceed with this action? [y/N]: y [enter]
[1/1] Fetching rsync-3.1.2_7.txz: 100% 298 KiB 305.3kB/s 00:01
Checking integrity... done (0 conflicting)
[1/1] Installing rsync-3.1.2_7...
Extracting rsync-3.1.2_7: 100%
[root@server /usr/home/user]#Configuration
List installed services with:
[root@server /usr/home/user]# service -r | grep rsync [enter]
/usr/local/etc/rc.d/rsyncd
[root@server /usr/home/user]#Find the rcvar for /etc/rc.conf:
[root@server /usr/home/user]# /usr/local/etc/rc.d/rsyncd rcvar [enter]
# rsyncd
#
rsyncd_enable="NO"
# (default: "")
[root@server /usr/home/user]#To start rsync automatically on system boot with –ipv4 as the prefered option add information to /etc/rc.conf with this commands:
[root@server /usr/home/user]# echo '' >> /etc/rc.conf; echo '# rsync using IPv4' >> /etc/rc.conf; echo 'rsyncd_enable="YES"' >> /etc/rc.conf; echo 'rsyncd_flags="--ipv4"' >> /etc/rc.conf [enter]
[root@server /usr/home/user]#If you run rsyncd manually and your server only uses IPv4, then make sure you add the “–ipv4” argument to the manual start command_args with:
[root@server /usr/home/user]# perl -pi -e 's/--daemon/--ipv4 --daemon/g' /usr/local/etc/rc.d/rsyncd [enter]
[root@server /usr/home/user]#Create a logfile with:
[root@server /usr/home/user]# touch /var/log/rsyncd.log [enter]
[root@server /usr/home/user]#Automatically rotate /var/log/rsyncd.log log file with:
[root@server /usr/home/user]# echo '/var/log/rsyncd.log 600 9 100000 * Z' >> /etc/newsyslog.conf [enter]
[root@server /usr/home/user]#Create the rsync Secret File with:
[root@server /usr/home/user]# ee /usr/local/etc/rsync/rsyncd.secrets [enter]Add the following text, example:
tridge:passwd1
susan:passwd2Make file /usr/local/etc/rsync/rsyncd.secrets non-world readable with:
[root@server /usr/home/user]# chmod 440 /usr/local/etc/rsync/rsyncd.secrets [enter]
[root@server /usr/home/user]#…and then set owner and group with:
[root@server /usr/home/user]# chown root:wheel /usr/local/etc/rsync/rsyncd.secrets [enter]
[root@server /usr/home/user]#Create group rsync and user rsync with:
[root@server /usr/home/user]# pw group add -n rsync -g 4002; pw user add -n rsync -u 4002 -c "rsync daemon" -d /nonexistent -s /usr/sbin/nologin [enter]
[root@server /usr/home/user]#The rsync group is added to /etc/group and should look similar to the following:
[root@server /usr/home/user]# grep rsync /etc/group [enter]
rsync:*:4002:
[root@server /usr/home/user]#The rsync user is added to /etc/passwd and should look similar to the following:
[root@server /usr/home/user]# grep rsync /etc/passwd [enter]
rsync:*:4002:4002:rsync daemon:/nonexistent:/usr/sbin/nologin
[root@server /usr/home/user]#Edit file /usr/local/etc/rsync/rsyncd.conf with:
[root@server /usr/home/user]# ee /usr/local/etc/rsync/rsyncd.conf [enter]Example:
# rsyncd.conf - Example file, see rsyncd.conf(5)
#
# Set this if you want to stop rsync daemon with rc.d scripts
pid file = /var/run/rsyncd.pid
# Edit this file before running rsync daemon!!
#uid = rsync
#gid = rsync
#use chroot = no
#max connections = 4
#syslog facility = local5
#[ftp]
# path = /var/ftp/pub
# comment = whole ftp area (approx 6.1 GB)
#[sambaftp]
# path = /var/ftp/pub/samba
# comment = Samba ftp area (approx 300 MB)
#[rsyncftp]
# path = /var/ftp/pub/rsync
# comment = rsync ftp area (approx 6 MB)
#[sambawww]
# path = /public_html/samba
# comment = Samba WWW pages (approx 240 MB)
#[cvs]
# path = /data/cvs
# comment = CVS repository (requires authentication)
# auth users = tridge, susan
# secrets file = /usr/local/etc/rsync/rsyncd.secretsManually Start
Manualy start the rsyncd with:
[root@server /usr/home/user]# service rsyncd start [enter]
Starting rsyncd.
[root@server /usr/home/user]#View rsyncd status with:
[root@server /usr/home/user]# service rsyncd status [enter]
rsyncd is running as pid 902.
[root@server /usr/home/user]#[root@server /usr/home/user]# ps aux | grep rsync [enter]
root 3527 0.0 0.0 12808 2448 - Ss 10:36PM 0:00.00 /usr/local/bin/rsync --ipv4 --ipv4 --daemon --config /usr/local/etc/
root 3535 0.0 0.0 18824 2332 0 S+ 10:37PM 0:00.00 grep rsync
[root@server /usr/home/user]#[root@server /usr/home/user]# sockstat | grep rsync [enter]
root rsync 3527 3 dgram -> /var/run/logpriv
root rsync 3527 4 tcp4 *:873 *:*
[root@server /usr/home/user]#Verify that you can connect to the daemon with:
[root@server /usr/home/user]# telnet localhost 873 [enter]
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
@RSYNCD: 30.0
[root@server /usr/home/user]#rsync Client Setup
N.B.: You may have to install rsync on the client as well!
Create a password file for user user with:
[root@server /usr/home/user]# echo "passwd" > /usr/local/etc/rsyncd.passwd_user [enter]
[root@server /usr/home/user]#Note: Echo password ONLY! Do NOT echo username!
Make file /usr/local/etc/rsyncd.passwd_user non-world readable with:
[root@server /usr/home/user]# chmod 440 /usr/local/etc/rsyncd.passwd_user [enter]
[root@server /usr/home/user]#Set owner and group with:
[root@server /usr/home/user]# chown root:wheel /usr/local/etc/rsyncd.passwd_user [enter]
[root@server /usr/home/user]#Howto use
You use rsync in the same way you use rcp. You must specify a source and a destination, one of which may be remote.
This is a syntax example for a manual file transfer from a remote host:
[root@server /usr/home/user]# rsync -avz --delete --stats --safe-links --password-file=/usr/local/etc/rsyncd.passwd_user user@192.168.1.100::ftp /var/ftp/pub/ [enter]This is a syntax example for a manual file transfer to a remote host:
[root@server /usr/home/user]# rsync -avz --delete --stats --safe-links --password-file=/usr/local/etc/rsyncd.passwd_user /var/ftp/pub/ user@192.168.1.100::ftp [enter]rsync can execute commands on the remote computer to generate a list of files to copy. The shell command is expanded by the remote shell before rsync is called.
The following command will run a find command on the remote host in directory ‘/tmp/test’ and rsync all “txt” files it finds to directory ‘/temp/test/’ on the local host:
[root@server /usr/home/user]# rsync -avR ssh user@remotehost:'`find /tmp/test -name "*.[txt]"`' /tmp/test/ [enter]rsync to Remote Server without Password
No-password authentication works because of public key crypto. Let’s say you have a local machine server and a remote machine remote. You want to be able to ssh from server to remote without having to enter your password.
The server remote in this document is the server that has files that is to be transferred to the local server server.
The server server in this document is the local server that will receive files from the remote server remote.
First step is to prepare the remote server remote by generate a public/private RSA key pair.
Next, we generate a public/private RSA key pair on on the local server, server and then we send the public key to the remote server, remote, so that remote knows that the server key belongs to a list of authorized keys. Then when we try to ssh from server to remote, RSA authentication is performed automatically.
On the Remote Server remote:
Generate keys on the remote server, remote as the user that will be connect to from the local server server:
[user@remote ~]$ ssh-keygen -t dsa -f ~/.ssh/id_dsa [enter]NOTE: When prompted for a password, do NOT enter one, just press [enter]!
Generating public/private dsa key pair.
Enter file in which to save the key (/home/user/.ssh/id_dsa):
Enter passphrase (empty for no passphrase): [enter]
Enter same passphrase again: [enter]
Your identification has been saved in /home/user/.ssh/id_dsa.
Your public key has been saved in /home/user/.ssh/id_dsa.pub.
The key fingerprint is:
ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff user@remote.example.net
The key's randomart image is:
+--[ DSA 1024]----+
|.B+o . |
|.+= + . |
|o... . |
|.... |
| . .... S |
| .o+o . . |
| E += |
| ..o |
| . |
+-----------------+
[user@remote ~]$This will create folder /usr/home/.ssh if it do not exist, generate a password-less key /usr/home/.ssh/id_dsa, and a public key /usr/home/.ssh/id_dsa.pub.
On the Local Server server:
Generate keys on the local server, server as the user that will perform the ssh connection to the remote server:
[user@server ~]$ ssh-keygen -t dsa -f ~/.ssh/id_dsa [enter]NOTE: When prompted for a password, do NOT enter one, just press [enter]!
Generating public/private dsa key pair.
Enter file in which to save the key (/home/user/.ssh/id_dsa):
Enter passphrase (empty for no passphrase): [enter]
Enter same passphrase again: [enter]
Your identification has been saved in /home/user/.ssh/id_dsa.
Your public key has been saved in /home/user/.ssh/id_dsa.pub.
The key fingerprint is:
ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff user@server.example.net
The key's randomart image is:
+--[ DSA 1024]----+
|.B+o . |
|.+= + . |
|o... . |
|.... |
| . .... S |
| .o+o . . |
| E += |
| ..o |
| . |
+-----------------+
[user@server ~]$This will create folder /usr/home/.ssh if it do not exist, generate a password-less key /usr/home/.ssh/id_dsa, and a public key /usr/home/.ssh/id_dsa.pub.
Copy the id_dsa.pub key over to the remote server, remote:
[user@server ~]$ scp ~/.ssh/id_dsa.pub user@remote.example.net:~/.ssh/server.pub [enter]
The authenticity of host 'remote.example.net (192.168.1.3)' can't be established.
DSA key fingerprint is ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff.
Are you sure you want to continue connecting (yes/no)? yes [enter]
Warning: Permanently added 'remote.example.net' (DSA) to the list of known hosts.
Password: ******* [enter]
id_dsa.pub 100% 622 0.6KB/s 00:00
[user@server ~]$Next, log in to remote server, remote, as the user that will perform the ssh connections:
[user@server ~]$ ssh user@remote.example.net
Password: ******* [enter]
Last login: Thu May 22 20:47:16 2009 from server.example.net
Copyright (c) 1980, 1983, 1986, 1988, 1990, 1991, 1993, 1994
The Regents of the University of California. All rights reserved.
Welcome to remote.example.net/192.168.1.nn Running FreeBSD 11.0-RELEASE!
[user@remote ~]$Add the public key from server to the list of authorized keys on remote with:
[user@remote ~]$ cat ~/.ssh/server.pub >> ~/.ssh/authorized_keys [enter]
[user@remote ~]$..and to protect the file ‘authorized_keys file from beeng changed do:
[user@remote ~]$ chmod 640 ~/.ssh/authorized_keys [enter]
[user@remote ~]$Delete the transferd key file with:
[user@remote ~]$ rm -f ~/.ssh/server.pub [enter]
[user@remote ~]$At this point the remote server remote should accept a password-less login from local server server by the user user.
If the ssh connection is to be performed as user root the following extra configuration must be performed to permit root to login:
[root@server ~]$ su - [enter]
Password: ****** [enter]
[root@server /usr/home/user]#[root@server /usr/home/user]# ee /etc/ssh/sshd_config [enter]Go to line 45:
# Authentication:
#LoginGraceTime 2m
#PermitRootLogin no
#StrictModes yes
#MaxAuthTries 6…and edit line 45 to look like this:
# Authentication:
#LoginGraceTime 2m
PermitRootLogin yes
#StrictModes yes
#MaxAuthTries 6Save file /etc/sshd_config and exit to local server server with:
[root@server /usr/home/user]# exit [enter]
logout
[techpc@remote ~]$ exit [enter]
Connection to remote.example.net closed.
[user@server ~]$To verify that the password-less ssh login to the remote server remote from the local server, server, works:
[user@server ~]$ ssh user@remote.example.net…should – without any password request – result in somthing like this example:
Last login: Sat Feb 19 16:32:09 2011 from 192.168.1.101
Copyright (c) 1980, 1983, 1986, 1988, 1990, 1991, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 11.0-RELEASE (GENERIC) #0: Mon Jul 19 02:36:49 UTC 2010
Welcome to remote.example.net running FreeBSD 8.1-RELEASE!
[user@remote ~]$We have successfully logged on to remote!
Issue the following command to log out from remote server remote:
[user@remote ~]$ exit [enter]
Connection to remote.example.net closed.
[user@server ~]$To backup directory /srv/test on remote server remote to local server server issue the following command, example:
[user@server ~]$ /usr/local/bin/rsync -aquz -e "ssh -l user" remote.example.net:/srv/test/ /srv/test/rsync – synchronizing two file trees
This section describes how to use rsync to synchronize file trees on two servers.
Remote Server Setup
In this example, we’re going to be using a remote rsync server containing the file tree that we like to syncronize with.
On the remote server edit file /usr/local/etc/rsyncd.conf with:
[root@server /usr/home/user]# ee /usr/local/etc/rsyncd.conf [enter]Edit file, example:
#
# rsyncd.conf
#
uid = nobody
gid = nobody
use chroot = yes
max connections = 4
syslog facility = local5
pid file = /var/run/rsyncd.pid
[example]
path = /srv/example/
comment = all of the example
auth users = tridge, susan
secrets file = /usr/local/etc/rsyncd.secretsMake /usr/local/etc/rsyncd.conf non-world readable with:
[root@server /usr/home/user]# chmod 640 /usr/local/etc/rsyncd.conf [enter]
[root@server /usr/home/user]#On the remote server create file /usr/local/etc/rsyncd.secrets with:
[root@server /usr/home/user]# ee /usr/local/etc/rsyncd.secrets [enter]Edit file, example:
#
# rsyncd.secrets
#
tridge:mypass
susan:herpassMake /usr/local/etc/rsyncd.secrets non-world readable with:
[root@server /usr/home/user]# chmod 640 /usr/local/etc/rsyncd.secrets [enter]
[root@server /usr/home/user]#Local Server Setup
rsync should have been installed on the local server too.
Manually synchronize files using ssh with:
[root@server /usr/home/user]# rsync -e ssh -avz --delete susan@remote.example.net:example /srv/example [enter]
Password: ****** [enter]
receiving file list ... done
[root@server /usr/home/user]#