% Generated by Sphinx.
\def\sphinxdocclass{report}
\documentclass[letterpaper,10pt,english]{sphinxmanual}
\usepackage[utf8]{inputenc}
\DeclareUnicodeCharacter{00A0}{\nobreakspace}
\usepackage[T1]{fontenc}
\usepackage{babel}
\usepackage{times}
\usepackage[Bjarne]{fncychap}
\usepackage{longtable}
\usepackage{sphinx}
\usepackage{multirow}


\title{S3QL Documentation}
\date{April 09, 2013}
\release{1.14}
\author{Nikolaus Rath}
\newcommand{\sphinxlogo}{}
\renewcommand{\releasename}{Release}
\makeindex

\makeatletter
\def\PYG@reset{\let\PYG@it=\relax \let\PYG@bf=\relax%
    \let\PYG@ul=\relax \let\PYG@tc=\relax%
    \let\PYG@bc=\relax \let\PYG@ff=\relax}
\def\PYG@tok#1{\csname PYG@tok@#1\endcsname}
\def\PYG@toks#1+{\ifx\relax#1\empty\else%
    \PYG@tok{#1}\expandafter\PYG@toks\fi}
\def\PYG@do#1{\PYG@bc{\PYG@tc{\PYG@ul{%
    \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}}
\def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}}

\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}}
\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}}
\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.25,0.82}{##1}}}
\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf}
\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}}
\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.82,0.44,0.00}{##1}}}
\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.00,0.88}{##1}}}
\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.31,0.50}{##1}}}
\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}}
\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit}
\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.38,0.56}{##1}}}
\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.82}{##1}}}
\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}}
\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.31,0.44,0.56}{##1}}}
\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}}
\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.00}{##1}}}
\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.56,0.44,0.00}{##1}}}
\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}}
\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.19,0.38}{##1}}}
\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.75}{##1}}}
\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.69,0.00,0.38}{##1}}}
\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.31,0.31,0.31}{##1}}}
\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.94,0.00,0.00}{##1}}}
\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.38,0.69}{##1}}}
\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.88,0.88,0.88}{\strut ##1}}}
\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.69}{##1}}}
\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.00}{##1}}}
\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.56,0.38,0.19}{##1}}}
\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}}
\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}}
\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.82,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}}
\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}}
\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.00,0.88}{##1}}}
\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.94,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.94,0.63,0.63}{\strut ##1}}}
\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.38,0.00}{##1}}}
\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}}
\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.25,0.00,0.88}{##1}}}
\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.82}{##1}}}
\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.19}{##1}}}
\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.19,0.50}{##1}}}
\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}}
\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.56}{##1}}}
\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.25,0.82}{##1}}}
\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.38,0.38}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.82,0.25,0.13}{##1}}}

\def\PYGZbs{\char`\\}
\def\PYGZus{\char`\_}
\def\PYGZob{\char`\{}
\def\PYGZcb{\char`\}}
\def\PYGZca{\char`\^}
\def\PYGZam{\char`\&}
\def\PYGZlt{\char`\<}
\def\PYGZgt{\char`\>}
\def\PYGZsh{\char`\#}
\def\PYGZpc{\char`\%}
\def\PYGZdl{\char`\$}
\def\PYGZti{\char`\~}
% for compatibility with earlier versions
\def\PYGZat{@}
\def\PYGZlb{[}
\def\PYGZrb{]}
\makeatother

\begin{document}

\maketitle
\tableofcontents
\phantomsection\label{index::doc}



\chapter{About S3QL}
\label{about:about-s3ql}\label{about::doc}\label{about:s3ql-user-s-guide}
S3QL is a file system that stores all its data online using storage
services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3AmazonS3}{Amazon S3} or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL effectively provides
a hard disk of dynamic, infinite capacity that can be accessed from
any computer with internet access running Linux, FreeBSD or OS-X.

S3QL is a standard conforming, full featured UNIX file system that is
conceptually indistinguishable from any local file system.
Furthermore, S3QL has additional features like compression,
encryption, data de-duplication, immutable trees and snapshotting
which make it especially suitable for online backup and archival.

S3QL is designed to favor simplicity and elegance over performance and
feature-creep. Care has been taken to make the source code as
readable and serviceable as possible. Solid error detection and error
handling have been included from the very first line, and S3QL comes
with extensive automated test cases for all its components.


\section{Features}
\label{about:features}\begin{itemize}
\item {} 
\textbf{Transparency.} Conceptually, S3QL is indistinguishable from a
local file system. For example, it supports hardlinks, symlinks,
standard unix permissions, extended attributes and file
sizes up to 2 TB.

\item {} 
\textbf{Dynamic Size.} The size of an S3QL file system grows and shrinks
dynamically as required.

\item {} 
\textbf{Compression.} Before storage, all data may compressed with the
LZMA, bzip2 or deflate (gzip) algorithm.

\item {} 
\textbf{Encryption.} After compression (but before upload), all data can
AES encrypted with a 256 bit key. An additional SHA256 HMAC checksum
is used to protect the data against manipulation.

\item {} 
\textbf{Data De-duplication.} If several files have identical contents,
the redundant data will be stored only once. This works across all
files stored in the file system, and also if only some parts of the
files are identical while other parts differ.

\item {} 
\textbf{Immutable Trees.} Directory trees can be made immutable, so that
their contents can no longer be changed in any way whatsoever. This
can be used to ensure that backups can not be modified after they
have been made.

\item {} 
\textbf{Copy-on-Write/Snapshotting.} S3QL can replicate entire directory
trees without using any additional storage space. Only if one of the
copies is modified, the part of the data that has been modified will
take up additional storage space. This can be used to create
intelligent snapshots that preserve the state of a directory at
different points in time using a minimum amount of space.

\item {} 
\textbf{High Performance independent of network latency.} All operations
that do not write or read file contents (like creating directories
or moving, renaming, and changing permissions of files and
directories) are very fast because they are carried out without any
network transactions.

S3QL achieves this by saving the entire file and directory structure
in a database. This database is locally cached and the remote
copy updated asynchronously.

\item {} 
\textbf{Support for low bandwidth connections.} S3QL splits file contents
into smaller blocks and caches blocks locally. This minimizes both
the number of network transactions required for reading and writing
data, and the amount of data that has to be transferred when only
parts of a file are read or written.

\end{itemize}


\section{Development Status}
\label{about:development-status}
After two years of beta-testing by about 93 users did not reveal any
data-critical bugs, S3QL was declared \textbf{stable} with the release of
version 1.0 on May 13th, 2011. Note that this does not mean that S3QL
is bug-free. S3QL still has several known, and probably many more
unknown bugs. However, there is a high probability that these bugs
will, although being inconvenient, not endanger any stored data.

Please report any problems on the \href{http://groups.google.com/group/s3ql}{mailing list} or the \href{http://code.google.com/p/s3ql/issues/list}{issue tracker}.


\chapter{Installation}
\label{installation:installation}\label{installation::doc}
S3QL depends on several other programs and libraries that have to be
installed first. The best method to satisfy these dependencies depends
on your distribution. In some cases S3QL and all its dependencies can
be installed with as little as three commands, while in other cases more work
may be required.

The \href{http://code.google.com/p/s3ql/w/list}{S3QL Wiki} contains
installation instructions for quite a few different Linux
distributions. You should only use the generic instructions in this
manual if your distribution is not included in the \href{http://code.google.com/p/s3ql/w/list?q=label:Installation}{distribution-specific
installation instructions} on the wiki.


\section{Dependencies}
\label{installation:dependencies}
The following is a list of the programs and libraries required for
running S3QL. Generally, you should first check if your distribution
already provides a suitable packages and only install from source if
that is not the case.
\begin{itemize}
\item {} 
Kernel: Linux 2.6.9 or newer or FreeBSD with \href{http://www.freshports.org/sysutils/fusefs-kmod/}{FUSE4BSD}. Starting with
kernel 2.6.26 you will get significantly better write performance,
so under Linux you should actually use \emph{2.6.26 or newer whenever
possible}.

\item {} 
\href{http://www.python.org/}{Python} 2.7.0 or newer (but not Python
3.x). Make sure to also install the development headers.

\item {} 
The \href{http://pypi.python.org/pypi/pycryptopp}{PyCrypto++ Python Module}. To check if this module
is installed, try to execute \code{python -c 'import pycryptopp'}.

\item {} 
\href{http://www.sqlite.org/}{SQLite} version 3.7.0 or newer. SQLite
has to be installed as a \emph{shared library} with development headers.

\item {} 
The \href{http://code.google.com/p/apsw/}{APSW Python Module}. To check
which (if any) version of APWS is installed, run the command

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{python -c 'import apsw; print apsw.apswversion()'}
\end{Verbatim}

The printed version number should be at least 3.7.0. Note that APSW
must be linked \emph{dynamically} against SQLite, so you can \emph{not} use
the Ubuntu PPA at
\href{https://launchpad.net/~ubuntu-rogerbinns/+archive/apsw}{https://launchpad.net/\textasciitilde{}ubuntu-rogerbinns/+archive/apsw} (these
packages are statically linked).

\item {} 
The \href{http://pypi.python.org/pypi/pyliblzma}{PyLibLZMA Python module}. To check if this module
is installed, execute \code{python -c 'import lzma; print
lzma.\_\_version\_\_'}. This should print a version number. You need at
least version 0.5.3.

\item {} 
The \href{http://code.google.com/p/python-llfuse/}{Python LLFUSE module}. To check if this module
is installed, execute \code{python -c 'import llfuse; print
llfuse.\_\_version\_\_'}. This should print a version number. You need at
least version 0.37.

\end{itemize}


\section{Installing S3QL}
\label{installation:inst-s3ql}\label{installation:installing-s3ql}
To install S3QL itself, proceed as follows:
\begin{enumerate}
\item {} 
Download S3QL from \href{http://code.google.com/p/s3ql/downloads/list}{http://code.google.com/p/s3ql/downloads/list}

\item {} 
Unpack it into a folder of your choice

\item {} 
Run \code{python setup.py build} to build S3QL.

\item {} 
Run \code{python setup.py test} to run a self-test. If this fails, ask
for help on the \href{http://groups.google.com/group/s3ql}{mailing list} or report a bug in the
\href{http://code.google.com/p/s3ql/issues/list}{issue tracker}.

\end{enumerate}

Now you have three options:
\begin{itemize}
\item {} 
You can run the S3QL commands from the \code{bin/} directory.

\item {} 
You can install S3QL system-wide for all users. To do that, you
have to run \code{sudo python setup.py install}.

\item {} 
You can install S3QL into \code{\textasciitilde{}/.local} by executing \code{python
setup.py install -{-}user}. In this case you should make sure that
\code{\textasciitilde{}/.local/bin} is in your \code{\$PATH} environment variable.

\end{itemize}


\section{Development Version}
\label{installation:development-version}
If you have checked out the unstable development version from the
Mercurial repository, a bit more effort is required. You need to also
have \href{http://www.cython.org/}{Cython} (0.16 or newer) and \href{http://sphinx.pocoo.org/}{Sphinx} (1.1 or newer) installed, and
the necessary commands are:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{python setup.py build\PYGZus{}cython}
\PYG{l}{python setup.py build\PYGZus{}ext --inplace}
\PYG{l}{python setup.py build\PYGZus{}sphinx}
\PYG{l}{python setup.py test}
\PYG{l}{python setup.py install}
\end{Verbatim}


\chapter{Storage Backends}
\label{backends::doc}\label{backends:id1}\label{backends:sphinx}\label{backends:storage-backends}
S3QL supports different \emph{backends} to store data at different service
providers and using different protocols. A \emph{storage url} specifies a
backend together with some backend-specific information and uniquely
identifies an S3QL file system. The form of the storage url depends on
the backend and is described for every backend below.

All storage backends respect the \code{http\_proxy} and \code{https\_proxy}
environment variables.


\section{Google Storage}
\label{backends:google-storage}
\href{http://code.google.com/apis/storage/}{Google Storage} is an online
storage service offered by Google. To use the Google Storage backend,
you need to have (or sign up for) a Google account, and then \href{http://code.google.com/apis/storage/docs/signup.html}{activate
Google Storage}
for your account. The account is free, you will pay only for the
amount of storage and traffic that you actually use. Once you have
created the account, make sure to \href{http://code.google.com/apis/storage/docs/reference/v1/apiversion1.html\#enabling}{activate legacy access}.

To create a Google Storage bucket, you can use e.g. the \href{https://sandbox.google.com/storage/}{Google
Storage Manager}. The storage
URL for accessing the bucket in S3QL is then

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{gs://}\PYG{n+nv}{\PYGZlt{}bucketname\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}prefix\PYGZgt{}}
\end{Verbatim}

Here \emph{bucketname} is the name of the bucket, and \emph{prefix} can be
an arbitrary prefix that will be prepended to all object names used by
S3QL. This allows you to store several S3QL file systems in the same
Google Storage bucket.

Note that the backend login and password for accessing your Google
Storage bucket are not your Google account name and password, but the
\emph{Google Storage developer access key} and \emph{Google Storage developer
secret} that you can manage with the \href{https://code.google.com/apis/console/\#:storage:legacy}{Google Storage key management
tool}.


\section{Amazon S3}
\label{backends:amazon-s3}
\href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service
offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To
use the S3 backend, you first need to sign up for an AWS account. The
account is free, you will pay only for the amount of storage and
traffic that you actually use. After that, you need to create a bucket
that will hold the S3QL file system, e.g. using the \href{https://console.aws.amazon.com/s3/home}{AWS Management
Console}. For best
performance, it is recommend to create the bucket in the
geographically closest storage region, but not the US Standard region
(see below).

The storage URL for accessing S3 buckets in S3QL has the form

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3://}\PYG{n+nv}{\PYGZlt{}bucketname\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}prefix\PYGZgt{}}
\end{Verbatim}

Here \emph{bucketname} is the name of the bucket, and \emph{prefix} can be an
arbitrary prefix that will be prepended to all object names used by
S3QL. This allows you to store several S3QL file systems in the same
S3 bucket.

Note that the backend login and password for accessing S3 are not the
user id and password that you use to log into the Amazon Webpage, but
the \emph{AWS access key id} and \emph{AWS secret access key} shown under \href{https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8\&action=access-key}{My
Account/Access Identifiers}.


\subsection{Reduced Redundancy Storage (RRS)}
\label{backends:reduced-redundancy-storage-rrs}
S3QL does not allow the use of \href{http://aws.amazon.com/s3/\#protecting}{reduced redundancy storage}. The reason for that is a
combination of three factors:
\begin{itemize}
\item {} 
RRS has a relatively low reliability, on average you lose one
out of every ten-thousand objects a year. So you can expect to
occasionally lose some data.

\item {} 
When \code{fsck.s3ql} asks S3 for a list of the stored objects, this list
includes even those objects that have been lost. Therefore
\code{fsck.s3ql} \emph{can not detect lost objects} and lost data will only
become apparent when you try to actually read from a file whose data
has been lost. This is a (very unfortunate) peculiarity of Amazon
S3.

\item {} 
Due to the data de-duplication feature of S3QL, unnoticed lost
objects may cause subsequent data loss later in time (see
{\hyperref[durability:backend-reliability]{\emph{Data Durability}}} for details).

\end{itemize}


\section{OpenStack/Swift}
\label{backends:openstack-swift}
\href{http://www.openstack.org/}{OpenStack} is an open-source cloud server application suite. \href{http://openstack.org/projects/storage/}{Swift} is
the cloud storage module of OpenStack. Swift/OpenStack storage is
offered by many different companies.

The storage URL for the OpenStack backend has the form

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{swift://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{g+ge}{[:\PYGZlt{}port\PYGZgt{}]}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]}
\end{Verbatim}

Note that the storage container must already exist. Most OpenStack
providers offer a web frontend that you can use to create storage
containers. \emph{prefix} can be an arbitrary prefix that will be prepended
to all object names used by S3QL. This allows you to store several
S3QL file systems in the same container.

The OpenStack backend always uses HTTPS connections. Note, however,
that at this point S3QL does not verify the server certificate (cf.
\href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}).


\section{RackSpace CloudFiles}
\label{backends:rackspace-cloudfiles}\label{backends:swift}
\href{http://www.rackspace.com/}{RackSpace} CloudFiles uses OpenStack internally, so you can use the
OpenStack/Swift backend (see above). The hostname for CloudFiles
containers is \code{auth.api.rackspacecloud.com}. Use your normal
RackSpace user name for the backend login, and your RackSpace API key
as the backend passphrase. You can create a storage container for S3QL
using the \href{https://manage.rackspacecloud.com/}{Control Panel} (go to
\emph{Cloud Files} under \emph{Hosting}).

\begin{notice}{warning}{Warning:}
As of January 2012, RackSpace does not give any information about
data consistency or data durability on their web page. However,
RackSpace support agents (especially in the live chat) often claim
very high guarantees. Any such statement is wrong. As of 01/2012,
RackSpace CloudFiles does \emph{not} give \emph{any} durability or
consistency guarantees (see {\hyperref[durability:durability]{\emph{Important Rules to Avoid Losing Data}}} for why this is
important). Why this fact is only acknowledged RackSpace's
technical engineers, and/or not communicated to their sales agents
is not known.
\end{notice}

You should note that opinions about RackSpace differ widely among S3QL
users and developers. On one hand, people praise RackSpace for their
backing of the (open source) OpenStack project. On the other hand,
their heavily advertised ``fanatical support'' is in practice often not
only \href{http://code.google.com/p/s3ql/issues/detail?id=243\#c5}{less than helpful}, but their
support agents also seem to be \href{http://code.google.com/p/s3ql/issues/detail?id=243\#c11}{downright incompetent}. However,
there are reports that the support quality increases dramatically once
you are a customer and use the ``Live Chat'' link when you are logged
into the control panel.


\section{S3 compatible}
\label{backends:s3-compatible}\label{backends:rackspace}
The S3 compatible backend allows S3QL to access any storage service
that uses the same protocol as Amazon S3. The storage URL has the form

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3c://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{l}{:}\PYG{n+nv}{\PYGZlt{}port\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}bucketname\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}prefix\PYGZgt{}}
\end{Verbatim}

Here \emph{bucketname} is the name of an (existing) bucket, and \emph{prefix}
can be an arbitrary prefix that will be prepended to all object names
used by S3QL. This allows you to store several S3QL file systems in
the same bucket.


\section{Local}
\label{backends:local}
S3QL is also able to store its data on the local file system. This can
be used to backup data on external media, or to access external
services that S3QL can not talk to directly (e.g., it is possible to
store data over SSH by first mounting the remote system using {\color{red}\bfseries{}{}`sshfs{}`\_}
and then using the local backend to store the data in the sshfs
mountpoint).

The storage URL for local storage is

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{local://}\PYG{n+nv}{\PYGZlt{}path\PYGZgt{}}
\end{Verbatim}

Note that you have to write three consecutive slashes to specify an
absolute path, e.g. \code{local:///var/archive}. Also, relative paths will
automatically be converted to absolute paths before the authentication
file (see {\hyperref[authinfo:authinfo]{\emph{Storing Authentication Information}}}) is read, i.e. if you are in the
\code{/home/john} directory and try to mount \code{local://s3ql}, the
corresponding section in the authentication file must match the
storage url \code{local:///home/john/s3ql}.


\chapter{Important Rules to Avoid Losing Data}
\label{durability:important-rules-to-avoid-losing-data}\label{durability::doc}\label{durability:durability}
Most S3QL backends store data in distributed storage systems. These
systems differ from a traditional, local hard disk in several
important ways. In order to avoid losing data, this section should be
read very carefully.


\section{Rules in a Nutshell}
\label{durability:rules-in-a-nutshell}
To avoid losing your data, obey the following rules:
\begin{enumerate}
\item {} 
Know what durability you can expect from your chosen storage
provider. The durability describes how likely it is that a stored
object becomes damaged over time. Such data corruption can never be
prevented completely, techniques like geographic replication and
RAID storage just reduce the likelihood of it to happen (i.e.,
increase the durability).

\item {} 
When choosing a backend and storage provider, keep in mind that
when using S3QL, the effective durability of the file system data
will be reduced because of S3QL's data de-duplication feature.

\item {} 
Determine your storage service's consistency window. The
consistency window that is important for S3QL is the smaller of the
times for which:
\begin{itemize}
\item {} 
a newly created object may not yet be included in the list of
stored objects

\item {} 
an attempt to read a newly created object may fail with the
storage service reporting that the object does not exist

\end{itemize}

If \emph{one} of the above times is zero, we say that as far as S3QL is
concerned the storage service has \emph{immediate} consistency.

If your storage provider claims that \emph{neither} of the above can
ever happen, while at the same time promising high durability, you
should choose a respectable provider instead.

\item {} 
When mounting the same file system on different computers (or on
the same computer but with different \code{-{-}cachedir} directories),
the time that passes between the first and second of invocation of
\textbf{mount.s3ql} must be at least as long as your storage
service's consistency window. If your storage service offers
immediate consistency, you do not need to wait at all.

\item {} 
Before running \textbf{fsck.s3ql} or \textbf{s3qladm}, the file system
must have been left untouched for the length of the consistency
window. If your storage service offers immediate consistency, you
do not need to wait at all.

\end{enumerate}

The rest of this section explains the above rules and the reasons for
them in more detail. It also contains a list of the consistency
windows for a number of larger storage providers.


\section{Consistency Window List}
\label{durability:consistency-window-list}
The following is a list of the consistency windows (as far as S3QL is
concerned) for a number of storage providers. This list doesn't come
with any guarantees and may be outdated. If your storage provider is
not included, or if you need more reliable information, check with
your storage provider.

\begin{tabulary}{\linewidth}{|L|L|}
\hline
\textbf{
Storage Provider
} & \textbf{
Consistency
}\\\hline

Amazon S3 in the US standard region
 & 
Eventual
\\\hline

Amazon S3 in other regions
 & 
Immediate
\\\hline

Google Storage
 & 
Immediate
\\\hline

RackSpace CloudFiles
 & 
Eventual
\\\hline
\end{tabulary}



\section{Data Consistency}
\label{durability:data-consistency}
In contrast to the typical hard disk, most storage providers do not
guarantee \emph{immediate consistency} of written data. This means that:
\begin{itemize}
\item {} 
after an object has been stored, requests to read this object may
still fail or return the prior contents for a little while.

\item {} 
after an object has been deleted, attempts to read it may still
return the (old) data for some time, and it may still remain in the
list of stored objects for some time.

\item {} 
after a new object has been created, it may still not be included
when retrieving the list of stored objects for some time.

\end{itemize}

Of course, none of this is acceptable for a file system, and S3QL
generally handles any of the above situations internally so that it
always provides a fully consistent file system to the user. However,
there are some situations where an S3QL user nevertheless needs to be
aware of the peculiarities of his chosen storage service.

Suppose that you mount the file system, store some new data, delete
some old data and unmount it. If you then mount the file system again
right away on another computer, there is no guarantee that S3QL will
see any of the changes that the first S3QL process has made. At least
in theory it is therefore possible that on the second mount, S3QL does
not see any of the changes that you have done and presents you an ``old
version'' of the file system without them. Even worse, if you notice
the problem and unmount the file system, S3QL will upload the old
status (which S3QL necessarily has to consider as current) and thereby
permanently override the newer version (even though this change may
not become immediately visible either). S3QL uses several techniques
to reduce the likelihood of this to happen (see {\hyperref[impl_details:impl-details]{\emph{Implementation Details}}}
for more information on this), but without support from the storage
service, the possibility cannot be eliminated completely.

The same problem of course also applies when checking the file system.
If the storage service provides S3QL with only partially updated data,
S3QL has no way to find out if this a real consistency problem that
needs to be fixed or if it is only a temporary problem that will
resolve itself automatically (because there are still changes that
have not become visible yet).

This is where the so called \emph{consistency window} comes in. The
consistency window is the maximum time (after writing or deleting the
object) for which any of the above ``outdated responses'' may be
received. If the consistency window is zero, i.e. all changes are
immediately effective, the storage service is said to have \emph{immediate
consistency}. If the window is infinite, i.e. there is no upper bound
on the time it may take for changes to become effect, the storage
service is said to be \emph{eventually consistent}. Note that often there
are different consistency windows for the different operations. For
example, Google Storage offers immediate consistency when reading
data, but only eventual consistency for the list of stored objects.

To prevent the problem of S3QL working with an outdated copy of the
file system data, it is therefore sufficient to simply wait for the
consistency window to pass before mounting the file system again (or
running a file system check). The length of the consistency window
changes from storage service to storage service, and if your service
is not included in the list below, you should check the web page or
ask the technical support of your storage provider. The window that is
important for S3QL is the smaller of the times for which
\begin{itemize}
\item {} 
a newly created object may not yet be included in the list of
stored objects

\item {} 
an attempt to read a newly created object may fail with the
storage service reporting that the object does not exist

\end{itemize}

Unfortunately, many storage providers are hesitant to guarantee
anything but eventual consistency, i.e. the length of the consistency
window is potentially infinite. In that case you simply have to pick a
length that you consider ``safe enough''. For example, even though
Amazon is only guaranteeing eventual consistency, the ordinary
consistency window for data stored in S3 is just a few seconds, and
only in exceptional circumstances (i.e., core network outages) it may
rise up to hours (\href{http://forums.aws.amazon.com/message.jspa?messageID=38471\#38471}{source}).


\section{Data Durability}
\label{durability:backend-reliability}\label{durability:data-durability}
The durability of a storage service a measure of the average
probability of a storage object to become corrupted over time. The
lower the chance of data loss, the higher the durability. Storage
services like Amazon S3 claim to achieve a durability of up to
99.999999999\% over a year, i.e. if you store 100000000 objects for 100
years, you can expect that at the end of that time one object will be
corrupted or lost.

S3QL is designed to reduce redundancy and store data in the smallest
possible form. Therefore, S3QL is generally not able to compensate for
any such losses, and when choosing a storage service you should
carefully review if the offered durability matches your requirements.
When doing this, there are two factors that should be kept in mind.

Firstly, even though S3QL is not able to compensate for storage
service failures, it is able to detect them: when trying to access
data that has been lost or corrupted by the storage service, an IO
error will be returned and the mount point will become inaccessible to
ensure that the problem is noticed.

Secondly, the consequences of a data loss by the storage service can
be significantly more severe than you may expect because of S3QL's
data de-duplication feature: a data loss in the storage service at
time \emph{x} may cause data that is written \emph{after} time \emph{x} to be lost as
well. Consider the following scenario:
\begin{enumerate}
\item {} 
You store an important file in the S3QL file system.

\item {} 
The storage service loses the data blocks of this file. As long as you
do not access the file or run \textbf{fsck.s3ql}, S3QL is not
aware that the data has been lost by the storage service.

\item {} 
You save an additional copy of the important file in a different
location on the same S3QL file system.

\item {} 
S3QL detects that the contents of the new file are identical to the
data blocks that have been stored earlier. Since at this point S3QL
is not aware that these blocks have been lost by the storage service, it
does not save another copy of the file contents in the storage service but
relies on the (presumably) existing blocks instead.

\item {} 
Therefore, even though you saved another copy, you still do not
have a backup of the important file (since both copies refer to the
same data blocks that have been lost by the storage service).

\end{enumerate}

For some storage services, \textbf{fsck.s3ql} can mitigate this
effect. When \textbf{fsck.s3ql} runs, it asks the storage service
for a list of all stored objects. If objects are missing, it can then
mark the damaged files and prevent the problem from spreading forwards
in time. Figuratively speaking, this establishes a ``checkpoint'': data
loss that occurred before running \textbf{fsck.s3ql} can not affect
any file system operations that are performed after the check.
Unfortunately, many storage services only ``discover'' that objects are
missing or broken when the object actually needs to be retrieved. In
this case, \textbf{fsck.s3ql} will not learn anything by just
querying the list of objects.

In the future, \textbf{fsck.s3ql} will have an additional
``full-check'' mode, in which it attempts to retrieve every single
object. However, this is expected to be rather time consuming and
expensive. Therefore, it is generally a better choice to choose a
storage service where the expected data durability is so high that the
possibility of a lost object (and thus the need to run any full
checks) can be neglected over long periods of time.

To some degree, \textbf{fsck.s3ql} can mitigate this effect. When
used with the \code{-{-}full-check} option, \textbf{fsck.s3ql} asks the
storage service to look up every stored object. This way, S3QL learns
about any missing and, depending on the storage service, corrupted
objects. It can then mark the damaged files and prevent the problem
from spreading forwards in time. Figuratively speaking, this
establishes a ``checkpoint'': data loss that occurred before running
\textbf{fsck.s3ql} with \code{-{-}full-check} can not affect any file
system operations that are performed after the check.

Unfortunately, a full check is rather time consuming and expensive
because of the need to check every single stored object. It is
generally a better choice to choose a storage service where the
expected data durability is so high that the possibility of a lost
object (and thus the need to run any full checks) can be neglected
over long periods of time.


\chapter{File System Creation}
\label{mkfs::doc}\label{mkfs:file-system-creation}
A S3QL file system is created with the \code{mkfs.s3ql} command. It has the
following syntax:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mkfs.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}

This command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default:
\code{\textasciitilde{}/.s3ql)} (default: /home/nikratio/.s3ql)
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file
(default: \code{\textasciitilde{}/.s3ql/authinfo2)} (default:
/home/nikratio/.s3ql/authinfo2)
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to
get debug messages from all modules. This option can
be specified multiple times. (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}ssl]  
Always use SSL connections when connecting to remote
servers. For backends that allow only encrypted
connections, S3QL uses SSL automatically, even if this
option is not set. (default: False)
\item [-{-}version]  
just print program version and exit
\item [-L \textless{}name\textgreater{}]  
Filesystem label (default: )
\item [-{-}max-obj-size \textless{}size\textgreater{}]  
Maximum size of storage objects in KiB. Files bigger
than this will be spread over multiple objects in the
storage backend. Default: 10240 KiB.
\item [-{-}plain]  
Create unencrypted file system. (default: False)
\item [-{-}force]  
Overwrite any existing data. (default: False)
\end{optionlist}
\end{quote}

Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask
you to enter an encryption password. This password will \emph{not} be read
from an authentication file specified with the \code{-{-}authfile}
option to prevent accidental creation of an encrypted file system.


\chapter{Managing File Systems}
\label{adm:managing-file-systems}\label{adm::doc}
The \code{s3qladm} command performs various operations on \emph{unmounted} S3QL
file systems. The file system \emph{must not be mounted} when using
\code{s3qladm} or things will go wrong badly.

The syntax is

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage-url\PYGZgt{}}
\end{Verbatim}

where \code{action} may be either of \textbf{passphrase},
\textbf{upgrade}, \textbf{clear} or \textbf{download-metadata}.

The \textbf{s3qladm} accepts the following general options, no
matter what specific action is being invoked:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get
debug messages from all modules. This option can be
specified multiple times. (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be rotated
when it reaches 1 MiB, and at most 5 old log files will
be kept. Specify \code{none} to disable logging. Default:
\code{none}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file (default:
\code{\textasciitilde{}/.s3ql/authinfo2)} (default:
/home/nikratio/.s3ql/authinfo2)
\item [-{-}ssl]  
Always use SSL connections when connecting to remote
servers. For backends that allow only encrypted
connections, S3QL uses SSL automatically, even if this
option is not set. (default: False)
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)}
(default: /home/nikratio/.s3ql)
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}

Hint: run \code{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional arguments
that the different actions take.


\section{Changing the Passphrase}
\label{adm:changing-the-passphrase}
To change the passphrase of a file system, use the \code{passphrase}
subcommand:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm passphrase  }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}


\section{Upgrading the file system}
\label{adm:upgrading-the-file-system}
If you have installed a new version of S3QL, it may sometimes be
necessary to upgrade the file system metadata as well. Note that in
this case the file system can no longer be accessed with older
versions of S3QL after the upgrade.

During the upgrade you have to make sure that the command is not
interrupted, and that no one else tries to mount, check or upgrade the
file system at the same time.

To upgrade a file system from the previous to the current revision,
execute

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm upgrade }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}


\section{Deleting a file system}
\label{adm:deleting-a-file-system}
A file system can be deleted with:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm clear }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}

This physically deletes all the data and file system structures.


\section{Restoring Metadata Backups}
\label{adm:restoring-metadata-backups}
If the most-recent copy of the file system metadata has been damaged
irreparably, it is possible to restore one of the automatically
created backup copies.

The command

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm download-metadata }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}

will give you a list of the available metadata backups and allow you
to download them. This will create two new files in the current
directory, ending in \code{.db} and \code{.params}. To actually use the
downloaded backup, you need to move these files into the \code{\textasciitilde{}/.s3ql/}
directory and run \code{fsck.s3ql}.

\begin{notice}{warning}{Warning:}
You should probably not use this functionality without having asked
for help on the mailing list first (see {\hyperref[resources:resources]{\emph{Further Resources / Getting Help}}}).
\end{notice}


\chapter{Mounting}
\label{mount:mounting}\label{mount::doc}
A S3QL file system is mounted with the \code{mount.s3ql} command. It has
the following syntax:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}
\end{Verbatim}

\begin{notice}{note}{Note:}
S3QL is not a network file system like \href{http://en.wikipedia.org/wiki/Network\_File\_System\_\%28protocol\%29}{NFS}
or \href{http://en.wikipedia.org/wiki/CIFS}{CIFS}. It can only be
mounted on one computer at a time.
\end{notice}

This command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be
rotated when it reaches 1 MiB, and at most 5 old log
files will be kept. Specify \code{none} to disable
logging. Default: \code{\textasciitilde{}/.s3ql/mount.log}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default:
\code{\textasciitilde{}/.s3ql)} (default: /home/nikratio/.s3ql)
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file
(default: \code{\textasciitilde{}/.s3ql/authinfo2)} (default:
/home/nikratio/.s3ql/authinfo2)
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to
get debug messages from all modules. This option can
be specified multiple times. (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}ssl]  
Always use SSL connections when connecting to remote
servers. For backends that allow only encrypted
connections, S3QL uses SSL automatically, even if this
option is not set. (default: False)
\item [-{-}version]  
just print program version and exit
\item [-{-}cachesize \textless{}size\textgreater{}]  
Cache size in KiB (default: 102400 (100 MiB)). Should
be at least 10 times the maximum object size of the
filesystem, otherwise an object may be retrieved and
written several times during a single write() or
read() operation. (default: 102400)
\item [-{-}max-cache-entries \textless{}num\textgreater{}]  
Maximum number of entries in cache (default: 768).
Each cache entry requires one file descriptor, so if
you increase this number you have to make sure that
your process file descriptor limit (as set with
\code{ulimit -n}) is high enough (at least the number of
cache entries + 100).
\item [-{-}allow-other]  
Normally, only the user who called \code{mount.s3ql} can
access the mount point. This user then also has full
access to it, independent of individual file
permissions. If the \code{-{-}allow-other} option is
specified, other users can access the mount point as
well and individual file permissions are taken into
account for all users. (default: False)
\item [-{-}allow-root]  
Like \code{-{-}allow-other}, but restrict access to the
mounting user and the root user. (default: False)
\item [-{-}fg]  
Do not daemonize, stay in foreground (default: False)
\item [-{-}single]  
Run in single threaded mode. If you don't understand
this, then you don't need it. (default: False)
\item [-{-}upstart]  
Stay in foreground and raise SIGSTOP once mountpoint
is up. (default: False)
\item [-{-}profile]  
Create profiling information. If you don't understand
this, then you don't need it. (default: False)
\item [-{-}compress \textless{}name\textgreater{}]  
Compression algorithm to use when storing new data.
Allowed values: \code{lzma}, \code{bzip2}, \code{zlib}, none.
(default: \code{lzma})
\item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}]  
Interval in seconds between complete metadata uploads.
Set to 0 to disable. Default: 24h. (default: 86400)
\item [-{-}threads \textless{}no\textgreater{}]  
Number of parallel upload threads to use (default:
auto). (default: None)
\item [-{-}nfs]  
Enable some optimizations for exporting the file
system over NFS. (default: False)
\end{optionlist}
\end{quote}


\section{Compression Algorithms}
\label{mount:compression-algorithms}
S3QL supports three compression algorithms, LZMA, Bzip2 and zlib (with
LZMA being the default). The compression algorithm can be specified
freely whenever the file system is mounted, since it affects only the
compression of new data blocks.

Roughly speaking, LZMA is slower but achieves better compression
ratios than Bzip2, while Bzip2 in turn is slower but achieves better
compression ratios than zlib.

For maximum file system performance, the best algorithm therefore
depends on your network connection speed: the compression algorithm
should be fast enough to saturate your network connection.

To find the optimal algorithm and number of parallel compression
threads for your system, S3QL ships with a program called
\code{benchmark.py} in the \code{contrib} directory. You should run this program
on a file that has a size that is roughly equal to the block size of
your file system and has similar contents. It will then determine the
compression speeds for the different algorithms and the upload speeds
for the specified backend and recommend the best algorithm that is
fast enough to saturate your network connection.

Obviously you should make sure that there is little other system load
when you run \code{benchmark.py} (i.e., don't compile software or encode
videos at the same time).


\section{Notes about Caching}
\label{mount:notes-about-caching}
S3QL maintains a local cache of the file system data to speed up
access. The cache is block based, so it is possible that only parts of
a file are in the cache.


\subsection{Maximum Number of Cache Entries}
\label{mount:maximum-number-of-cache-entries}
The maximum size of the cache can be configured with the \code{-{-}cachesize}
option. In addition to that, the maximum number of objects in the
cache is limited by the \code{-{-}max-cache-entries} option, so it is
possible that the cache does not grow up to the maximum cache size
because the maximum number of cache elements has been reached. The
reason for this limit is that each cache entry requires one open
file descriptor, and Linux distributions usually limit the total
number of file descriptors per process to about a thousand.

If you specify a value for \code{-{-}max-cache-entries}, you should therefore
make sure to also configure your system to increase the maximum number
of open file handles. This can be done temporarily with the \code{umask -n}
command. The method to permanently change this limit system-wide
depends on your distribution.


\subsection{Cache Flushing and Expiration}
\label{mount:cache-flushing-and-expiration}
S3QL flushes changed blocks in the cache to the backend whenever a block
has not been accessed for at least 10 seconds. Note that when a block is
flushed, it still remains in the cache.

Cache expiration (i.e., removal of blocks from the cache) is only done
when the maximum cache size is reached. S3QL always expires the least
recently used blocks first.


\section{Automatic Mounting}
\label{mount:automatic-mounting}
If you want to mount and umount an S3QL file system automatically at
system startup and shutdown, you should do so with one dedicated S3QL
init script for each S3QL file system.

If your system is using upstart, an appropriate job can be defined
as follows (and should be placed in \code{/etc/init/}):

\begin{Verbatim}[commandchars=\\\{\},numbers=left,firstnumber=1,stepnumber=1]
\PYG{l}{description	"S3QL Backup File System"}
\PYG{l}{author		"Nikolaus Rath }\PYG{n+nv}{\PYGZlt{}Nikolaus@rath.org\PYGZgt{}}\PYG{l}{"}

\PYG{l}{\PYGZsh{} This assumes that eth0 provides your internet connection}
\PYG{l}{start on (filesystem and net-device-up IFACE=eth0)}

\PYG{l}{\PYGZsh{} We can't use "stop on runlevel }\PYG{g+ge}{[016]}\PYG{l}{" because from that point on we}
\PYG{l}{\PYGZsh{} have only 10 seconds until the system shuts down completely.}
\PYG{l}{stop on starting rc RUNLEVEL=}\PYG{g+ge}{[016]}

\PYG{l}{\PYGZsh{} Time to wait before sending SIGKILL to the daemon and}
\PYG{l}{\PYGZsh{} pre-stop script}
\PYG{l}{kill timeout 300}

\PYG{l}{env STORAGE\PYGZus{}URL="s3://my-backup-bla"}
\PYG{l}{env MOUNTPOINT="/mnt/backup"}

\PYG{l}{env USER="myusername"}
\PYG{l}{env AUTHFILE="/path/to/authinfo2"}

\PYG{l}{expect stop}

\PYG{l}{script}
\PYG{l}{    \PYGZsh{} Redirect stdout and stderr into the system log}
\PYG{l}{    DIR=\PYGZdl{}(mktemp -d)}
\PYG{l}{    mkfifo "\PYGZdl{}DIR/LOG\PYGZus{}FIFO"}
\PYG{l}{    logger -t s3ql -p local0.info }\PYG{n+nv}{\PYGZlt{} "\PYGZdl{}DIR/LOG\PYGZus{}FIFO" \PYGZam{}}
\PYG{n+nv}{    exec \PYGZgt{}}\PYG{l}{ "\PYGZdl{}DIR/LOG\PYGZus{}FIFO"}
\PYG{l}{    exec 2\PYGZgt{}\PYGZam{}1}
\PYG{l}{    rm -rf "\PYGZdl{}DIR"}

\PYG{l}{    \PYGZsh{} Check and mount file system}
\PYG{l}{    su -s /bin/sh -c 'exec "\PYGZdl{}0" "\PYGZdl{}@"' "\PYGZdl{}USER" -- \PYGZbs{}}
\PYG{l}{        fsck.s3ql --batch --authfile "\PYGZdl{}AUTHFILE" "\PYGZdl{}STORAGE\PYGZus{}URL"}
\PYG{l}{    exec su -s /bin/sh -c 'exec "\PYGZdl{}0" "\PYGZdl{}@"' "\PYGZdl{}USER" -- \PYGZbs{}}
\PYG{l}{        mount.s3ql --upstart --authfile "\PYGZdl{}AUTHFILE" "\PYGZdl{}STORAGE\PYGZus{}URL" "\PYGZdl{}MOUNTPOINT"}
\PYG{l}{end script}

\PYG{l}{pre-stop script}
\PYG{l}{    su -s /bin/sh -c 'exec "\PYGZdl{}0" "\PYGZdl{}@"' "\PYGZdl{}USER" -- umount.s3ql "\PYGZdl{}MOUNTPOINT"}
\PYG{l}{end script}
\end{Verbatim}

\begin{notice}{note}{Note:}
In principle, it is also possible to automatically mount an S3QL
file system with an appropriate entry in \code{/etc/fstab}. However,
this is not recommended for several reasons:
\begin{itemize}
\item {} 
file systems mounted in \code{/etc/fstab} will be unmounted with the
\code{umount} command, so your system will not wait until all data has
been uploaded but shutdown (or restart) immediately (this is a
FUSE limitation, see \href{http://code.google.com/p/s3ql/issues/detail?id=159}{issue 159}).

\item {} 
There is no way to tell the system that mounting S3QL requires a
Python interpreter to be available, so it may attempt to run
\code{mount.s3ql} before it has mounted the volume containing the
Python interpreter.

\item {} 
There is no standard way to tell the system that internet
connection has to be up before the S3QL file system can be
mounted.

\end{itemize}
\end{notice}


\chapter{Advanced S3QL Features}
\label{special:advanced-s3ql-features}\label{special::doc}

\section{Snapshotting and Copy-on-Write}
\label{special:snapshotting-and-copy-on-write}\label{special:s3qlcp}
The command \code{s3qlcp} can be used to duplicate a directory tree without
physically copying the file contents. This is made possible by the
data de-duplication feature of S3QL.

The syntax of \code{s3qlcp} is:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}src\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}target\PYGZgt{}}
\end{Verbatim}

This will replicate the contents of the directory \code{\textless{}src\textgreater{}} in the
directory \code{\textless{}target\textgreater{}}. \code{\textless{}src\textgreater{}} has to be an existing directory and
\code{\textless{}target\textgreater{}} must not exist. Moreover, both directories have to be
within the same S3QL file system.

The replication will not take any additional space. Only if one of
directories is modified later on, the modified data will take
additional storage space.

\code{s3qlcp} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root})
the root user. This limitation might be removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).

Note that:
\begin{itemize}
\item {} 
After the replication, both source and target directory will still
be completely ordinary directories. You can regard \code{\textless{}src\textgreater{}} as a
snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the most common
usage of \code{s3qlcp} is to regularly duplicate the same source
directory, say \code{documents}, to different target directories. For a
e.g. monthly replication, the target directories would typically be
named something like \code{documents\_January} for the replication in
January, \code{documents\_February} for the replication in February etc.
In this case it is clear that the target directories should be
regarded as snapshots of the source directory.

\item {} 
Exactly the same effect could be achieved by an ordinary copy
program like \code{cp -a}. However, this procedure would be orders of
magnitude slower, because \code{cp} would have to read every file
completely (so that S3QL had to fetch all the data over the network
from the backend) before writing them into the destination folder.

\end{itemize}


\subsection{Snapshotting vs Hardlinking}
\label{special:snapshotting-vs-hardlinking}
Snapshot support in S3QL is inspired by the hardlinking feature that
is offered by programs like \href{http://www.samba.org/rsync}{rsync} or
\href{http://savannah.nongnu.org/projects/storebackup}{storeBackup}.
These programs can create a hardlink instead of copying a file if an
identical file already exists in the backup. However, using hardlinks
has two large disadvantages:
\begin{itemize}
\item {} 
backups and restores always have to be made with a special program
that takes care of the hardlinking. The backup must not be touched
by any other programs (they may make changes that inadvertently
affect other hardlinked files)

\item {} 
special care needs to be taken to handle files which are already
hardlinked (the restore program needs to know that the hardlink was
not just introduced by the backup program to safe space)

\end{itemize}

S3QL snapshots do not have these problems, and they can be used with
any backup program.


\section{Getting Statistics}
\label{special:s3qlstat}\label{special:getting-statistics}
You can get more information about a mounted S3QL file system with the
\code{s3qlstat} command. It has the following syntax:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlstat }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}
\end{Verbatim}

Probably the most interesting numbers are the total size of your data,
the total size after duplication, and the final size after
de-duplication and compression.

\code{s3qlstat} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root})
the root user. This limitation might be removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).

For a full list of available options, run \code{s3qlstat -{-}help}.


\section{Immutable Trees}
\label{special:immutable-trees}\label{special:s3qllock}
The command \textbf{s3qllock} can be used to make a directory tree
immutable. Immutable trees can no longer be changed in any way
whatsoever. You can not add new files or directories and you can not
change or delete existing files and directories. The only way to get
rid of an immutable tree is to use the \textbf{s3qlrm} command (see
below).

For example, to make the directory tree beneath the directory
\code{2010-04-21} immutable, execute

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qllock 2010-04-21}
\end{Verbatim}

Immutability is a feature designed for backups. Traditionally, backups
have been made on external tape drives. Once a backup was made, the
tape drive was removed and locked somewhere in a shelf. This has the
great advantage that the contents of the backup are now permanently
fixed. Nothing (short of physical destruction) can change or delete
files in the backup.

In contrast, when backing up into an online storage system like S3QL,
all backups are available every time the file system is mounted.
Nothing prevents a file in an old backup from being changed again
later on. In the worst case, this may make your entire backup system
worthless. Imagine that your system gets infected by a nasty virus
that simply deletes all files it can find -- if the virus is active
while the backup file system is mounted, the virus will destroy all
your old backups as well!

Even if the possibility of a malicious virus or trojan horse is
excluded, being able to change a backup after it has been made is
generally not a good idea. A common S3QL use case is to keep the file
system mounted at all times and periodically create backups with
\textbf{rsync -a}. This allows every user to recover her files from a
backup without having to call the system administrator. However, this
also allows every user to accidentally change or delete files \emph{in} one
of the old backups.

Making a backup immutable protects you against all these problems.
Unless you happen to run into a virus that was specifically programmed
to attack S3QL file systems, backups can be neither deleted nor
changed after they have been made immutable.


\section{Fast Recursive Removal}
\label{special:s3qlrm}\label{special:fast-recursive-removal}
The \code{s3qlrm} command can be used to recursively delete files and
directories on an S3QL file system. Although \code{s3qlrm} is faster than
using e.g. \code{rm -r}, the main reason for its existence is that it
allows you to delete immutable trees as well. The syntax is rather
simple:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlrm }\PYG{n+nv}{\PYGZlt{}directory\PYGZgt{}}
\end{Verbatim}

Be warned that there is no additional confirmation. The directory will
be removed entirely and immediately.


\section{Runtime Configuration}
\label{special:runtime-configuration}\label{special:s3qlctrl}
The \code{s3qlctrl} can be used to control a mounted S3QL file system. Its
syntax is

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ ...}
\end{Verbatim}

\code{\textless{}mountpoint\textgreater{}} must be the location of a mounted S3QL file system.
For a list of valid options, run \code{s3qlctrl -{-}help}. \code{\textless{}action\textgreater{}}
may be either of:
\begin{quote}
\begin{quote}\begin{description}
\item[{flushcache}] \leavevmode
Flush file system cache. The command blocks until the cache has
been flushed.

\item[{log}] \leavevmode
Change log level.

\item[{cachesize}] \leavevmode
Change file system cache size.

\item[{upload-meta}] \leavevmode
Trigger a metadata upload.

\end{description}\end{quote}
\end{quote}


\chapter{Unmounting}
\label{umount::doc}\label{umount:unmounting}
To unmount an S3QL file system, use the command:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{umount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}
\end{Verbatim}

This will block until all data has been written to the backend.

Only the user who mounted the file system with \textbf{mount.s3ql}
is able to unmount it again. If you are root and want to unmount an
S3QL file system mounted by an ordinary user, you have to use the
\textbf{fusermount -u} or \textbf{umount} command instead. Note
that these commands do not block until all data has been uploaded, so
if you use them instead of \code{umount.s3ql} then you should manually wait
for the \code{mount.s3ql} process to terminate before shutting down the
system.

The \textbf{umount.s3ql} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}version]  
just print program version and exit
\item [-{-}lazy, -z]  
Lazy umount. Detaches the file system immediately, even if
there are still open files. The data will be uploaded in the
background once all open files have been closed. (default:
False)
\end{optionlist}
\end{quote}

If, for some reason, the \code{umount.sql} command does not work, the file
system can also be unmounted with \code{fusermount -u -z}. Note that this
command will return immediately and the file system may continue to
upload data in the background for a while longer.


\chapter{Checking for Errors}
\label{fsck:checking-for-errors}\label{fsck::doc}
If, for some reason, the filesystem has not been correctly unmounted,
or if you suspect that there might be errors, you should run the
\code{fsck.s3ql} utility. It has the following syntax:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{fsck.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}

This command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be rotated
when it reaches 1 MiB, and at most 5 old log files will
be kept. Specify \code{none} to disable logging. Default:
\code{\textasciitilde{}/.s3ql/fsck.log}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)}
(default: /home/nikratio/.s3ql)
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file (default:
\code{\textasciitilde{}/.s3ql/authinfo2)} (default:
/home/nikratio/.s3ql/authinfo2)
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get
debug messages from all modules. This option can be
specified multiple times. (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}ssl]  
Always use SSL connections when connecting to remote
servers. For backends that allow only encrypted
connections, S3QL uses SSL automatically, even if this
option is not set. (default: False)
\item [-{-}version]  
just print program version and exit
\item [-{-}batch]  
If user input is required, exit without prompting.
(default: False)
\item [-{-}force]  
Force checking even if file system is marked clean.
(default: False)
\end{optionlist}
\end{quote}


\chapter{Storing Authentication Information}
\label{authinfo:storing-authentication-information}\label{authinfo:authinfo}\label{authinfo::doc}
Normally, S3QL reads username and password for the backend as well as
an encryption passphrase for the file system from the terminal. Most
commands also accept an \code{-{-}authfile} parameter that can be
used to read this information from a file instead.

The authentication file consists of sections, led by a \code{{[}section{]}}
header and followed by \code{name: value} entries. The section headers
themselves are not used by S3QL but have to be unique within the file.

In each section, the following entries can be defined:
\begin{quote}\begin{description}
\item[{storage-url}] \leavevmode
Specifies the storage url to which this section applies. If a
storage url starts with the value of this entry, the section is
considered applicable.

\item[{backend-login}] \leavevmode
Specifies the username to use for authentication with the backend.

\item[{backend-password}] \leavevmode
Specifies the password to use for authentication with the backend.

\item[{fs-passphrase}] \leavevmode
Specifies the passphrase to use to decrypt the file system (if
it is encrypted).

\end{description}\end{quote}

When reading the authentication file, S3QL considers every applicable
section in order and uses the last value that it found for each entry.
For example, consider the following authentication file:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+ge}{[s3]}
\PYG{l}{storage-url: s3://}
\PYG{l}{backend-login: joe}
\PYG{l}{backend-password: notquitesecret}

\PYG{g+ge}{[fs1]}
\PYG{l}{storage-url: s3://joes-first-bucket}
\PYG{l}{fs-passphrase: neitheristhis}

\PYG{g+ge}{[fs2]}
\PYG{l}{storage-url: s3://joes-second-bucket}
\PYG{l}{fs-passphrase: swordfish}

\PYG{g+ge}{[fs3]}
\PYG{l}{storage-url: s3://joes-second-bucket/with-prefix}
\PYG{l}{backend-login: bill}
\PYG{l}{backend-password: bi23ll}
\PYG{l}{fs-passphrase: ll23bi}
\end{Verbatim}

With this authentication file, S3QL would try to log in as ``joe''
whenever the s3 backend is used, except when accessing a storage url
that begins with ``s3://joes-second-bucket/with-prefix''. In that case,
the last section becomes active and S3QL would use the ``bill''
credentials. Furthermore, file system encryption passphrases will be used
for storage urls that start with ``s3://joes-first-bucket'' or
``s3://joes-second-bucket''.

The authentication file is parsed by the \href{http://docs.python.org/library/configparser.html}{Python ConfigParser
module}.


\chapter{Contributed Programs}
\label{contrib:contributed-programs}\label{contrib::doc}
S3QL comes with a few contributed programs that are not part of the
core distribution (and are therefore not installed automatically by
default), but which may nevertheless be useful. These programs are in
the \code{contrib} directory of the source distribution or in
\code{/usr/share/doc/s3ql/contrib} if you installed S3QL from a package.


\section{benchmark.py}
\label{contrib:benchmark-py}
This program measures S3QL write performance, uplink bandwidth and
compression speed to determine the limiting factor. It also gives
recommendation for compression algorithm and number of upload threads
to achieve maximum performance.


\section{s3\_copy.py}
\label{contrib:s3-copy-py}
This program physically duplicates Amazon S3 bucket. It can be used to
migrate buckets to a different storage region or storage class
(standard or reduced redundancy).


\section{pcp.py}
\label{contrib:pcp-py}\label{contrib:pcp}
\code{pcp.py} is a wrapper program that starts several rsync processes to
copy directory trees in parallel. This is important because
transferring files in parallel significantly enhances performance when
copying data from an S3QL file system (see {\hyperref[tips:copy-performance]{\emph{Improving copy performance}}} for
details).

To recursively copy the directory \code{/mnt/home-backup} into
\code{/home/joe} using 8 parallel processes and preserving permissions,
you would execute

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{pcp.py -a --processes=8 /mnt/home-backup/ /home/joe}
\end{Verbatim}


\section{s3\_backup.sh}
\label{contrib:s3-backup-sh}
This is an example script that demonstrates how to set up a simple but
powerful backup solution using S3QL and \href{http://samba.org/rsync}{rsync}.

The \code{s3\_backup.sh} script automates the following steps:
\begin{enumerate}
\item {} 
Mount the file system

\item {} 
Replicate the previous backup with {\hyperref[special:s3qlcp]{\emph{s3qlcp}}}

\item {} 
Update the new copy with the data from the backup source using rsync

\item {} 
Make the new backup immutable with {\hyperref[special:s3qllock]{\emph{s3qllock}}}

\item {} 
Delete old backups that are no longer needed

\item {} 
Unmount the file system

\end{enumerate}

The backups are stored in directories of the form
\code{YYYY-MM-DD\_HH:mm:SS} and the {\hyperref[contrib:expire-backups-py]{expire\_backups.py}} command is used to
delete old backups.


\section{expire\_backups.py}
\label{contrib:expire-backups-py}
\textbf{expire\_backups.py} is a program to intelligently remove old
backups that are no longer needed.

To define what backups you want to keep for how long, you define a
number of \emph{age ranges}. \textbf{expire\_backups} ensures that you
will have at least one backup in each age range at all times. It will
keep exactly as many backups as are required for that and delete any
backups that become redundant.

Age ranges are specified by giving a list of range boundaries in terms
of backup cycles. Every time you create a new backup, the existing
backups age by one cycle.

Example: when \textbf{expire\_backups} is called with the age range
definition \code{1 3 7 14 31}, it will guarantee that you always have the
following backups available:
\begin{enumerate}
\item {} 
A backup that is 0 to 1 cycles old (i.e, the most recent backup)

\item {} 
A backup that is 1 to 3 cycles old

\item {} 
A backup that is 3 to 7 cycles old

\item {} 
A backup that is 7 to 14 cycles old

\item {} 
A backup that is 14 to 31 cycles old

\end{enumerate}

\begin{notice}{note}{Note:}
If you do backups in fixed intervals, then one cycle will be
equivalent to the backup interval. The advantage of specifying the
age ranges in terms of backup cycles rather than days or weeks is
that it allows you to gracefully handle irregular backup intervals.
Imagine that for some reason you do not turn on your computer for
one month. Now all your backups are at least a month old, and if you
had specified the above backup strategy in terms of absolute ages,
they would all be deleted! Specifying age ranges in terms of backup
cycles avoids these sort of problems.
\end{notice}

\textbf{expire\_backups} usage is simple. It requires backups to have
names of the forms \code{year-month-day\_hour:minute:seconds}
(\code{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current
directory. So for the above backup strategy, the correct invocation
would be:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{expire\PYGZus{}backups.py 1 3 7 14 31}
\end{Verbatim}

When storing your backups on an S3QL file system, you probably want to
specify the \code{-{-}use-s3qlrm} option as well. This tells
\textbf{expire\_backups} to use the {\hyperref[special:s3qlrm]{\emph{s3qlrm}}} command to
delete directories.

\textbf{expire\_backups} uses a ``state file'' to keep track which
backups are how many cycles old (since this cannot be inferred from
the dates contained in the directory names). The standard name for
this state file is \code{.expire\_backups.dat}. If this file gets
damaged or deleted, \textbf{expire\_backups} no longer knows the ages
of the backups and refuses to work. In this case you can use the
\code{-{-}reconstruct-state} option to try to reconstruct the state
from the backup dates. However, the accuracy of this reconstruction
depends strongly on how rigorous you have been with making backups (it
is only completely correct if the time between subsequent backups has
always been exactly the same), so it's generally a good idea not to
tamper with the state file.

For a full list of available options, run \textbf{expire\_backups.py
--help}.


\section{s3ql\_upstart.conf}
\label{contrib:s3ql-upstart-conf}
\code{s3ql\_upstart.conf} is an example upstart job definition file. It
defines a job that automatically mounts an S3QL file system on system
start, and properly unmounts it when the system is shut down.


\chapter{Tips \& Tricks}
\label{tips:tips-tricks}\label{tips::doc}

\section{SSH Backend}
\label{tips:ssh-tipp}\label{tips:ssh-backend}
By combining S3QL's local backend with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, it is possible to store an
S3QL file system on arbitrary SSH servers: first mount the remote
target directory into the local filesystem,

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{sshfs user@my.server.com:/mnt/s3ql /mnt/sshfs}
\end{Verbatim}

and then give the mountpoint to S3QL as a local destination:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mount.s3ql local:///mnt/sshfs/myfsdata /mnt/s3ql}
\end{Verbatim}


\section{Permanently mounted backup file system}
\label{tips:permanently-mounted-backup-file-system}
If you use S3QL as a backup file system, it can be useful to mount the
file system permanently (rather than just mounting it for a backup and
unmounting it afterwards). Especially if your file system becomes
large, this saves you long mount- and unmount times if you only want
to restore a single file.

If you decide to do so, you should make sure to
\begin{itemize}
\item {} 
Use {\hyperref[special:s3qllock]{\emph{s3qllock}}} to ensure that backups are immutable
after they have been made.

\item {} 
Call {\hyperref[special:s3qlctrl]{\emph{s3qlctrl upload-meta}}} right after a every
backup to make sure that the newest metadata is stored safely (if
you do backups often enough, this may also allow you to set the
\code{-{-}metadata-upload-interval} option of \textbf{mount.s3ql}
to zero).

\end{itemize}


\section{Improving copy performance}
\label{tips:copy-performance}\label{tips:improving-copy-performance}
\begin{notice}{note}{Note:}
The following applies only when copying data \textbf{from} an S3QL file
system, \textbf{not} when copying data \textbf{to} an S3QL file system.
\end{notice}

If you want to copy a lot of smaller files \emph{from} an S3QL file system
(e.g. for a system restore) you will probably notice that the
performance is rather bad.

The reason for this is intrinsic to the way S3QL works. Whenever you
read a file, S3QL first has to retrieve this file over the network
from the backend. This takes a minimum amount of time (the network
latency), no matter how big or small the file is. So when you copy
lots of small files, 99\% of the time is actually spend waiting for
network data.

Theoretically, this problem is easy to solve: you just have to copy
several files at the same time. In practice, however, almost all unix
utilities (\code{cp}, \code{rsync}, \code{tar} and friends) insist on copying
data one file at a time. This makes a lot of sense when copying data
on the local hard disk, but in case of S3QL this is really
unfortunate.

The best workaround that has been found so far is to copy files by
starting several rsync processes at once and use exclusion rules to
make sure that they work on different sets of files.

For example, the following script will start 3 rsync instances. The
first instance handles all filenames starting with a-f, the second the
filenames from g-l and the third covers the rest. The \code{+ */} rule
ensures that every instance looks into all directories.

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{c}{\PYGZsh{}!/bin/bash}

\PYG{l}{RSYNC\PYGZus{}ARGS="-aHv /mnt/s3ql/ /home/restore/"}

\PYG{l}{rsync -f "+ */" -f "-! }\PYG{g+ge}{[a-f]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}}
\PYG{l}{rsync -f "+ */" -f "-! }\PYG{g+ge}{[g-l]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}}
\PYG{l}{rsync -f "+ */" -f "- }\PYG{g+ge}{[a-l]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}}

\PYG{l}{wait}
\end{Verbatim}

The optimum number of parallel processes depends on your network
connection and the size of the files that you want to transfer.
However, starting about 10 processes seems to be a good compromise
that increases performance dramatically in almost all situations.

S3QL comes with a script named \code{pcp.py} in the \code{contrib} directory
that can be used to transfer files in parallel without having to write
an explicit script first. See the description of {\hyperref[contrib:pcp]{\emph{pcp.py}}} for
details.


\chapter{Known Issues}
\label{issues:known-issues}\label{issues::doc}\begin{itemize}
\item {} 
S3QL does not support Access Control Lists (ACLs). This is due to
a bug in the FUSE library and will therefore hopefully be fixed
at some point. See \href{http://code.google.com/p/s3ql/issues/detail?id=385}{issue 385}
for more details.

\item {} 
S3QL does not verify TLS/SSL server certificates and is thus
vulnerable to man-in-the-middle attacks. See \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267} for more
details.

\item {} 
S3QL is rather slow when an application tries to write data in
unreasonably small chunks. If a 1 MiB file is copied in chunks of 1
KB, this will take more than 10 times as long as when it's copied
with the (recommended) chunk size of 128 KiB.

This is a limitation of the FUSE library (which does not yet support
write caching) which will hopefully be addressed in some future FUSE
version.

Most applications, including e.g. GNU \code{cp} and \code{rsync}, use
reasonably large buffers and are therefore not affected by this
problem and perform very efficient on S3QL file systems.

However, if you encounter unexpectedly slow performance with a
specific program, this might be due to the program using very small
write buffers. Although this is not really a bug in the program,
it might be worth to ask the program's authors for help.

\item {} 
S3QL always updates file and directory access times as if the \code{relatime}
mount option has been specified: the access time (``atime'') is only updated
if it is currently earlier than either the status change time
(``ctime'') or modification time (``mtime'').

\item {} 
S3QL directories always have an \code{st\_nlink} value of 1. This may confuse
programs that rely on directories having \code{st\_nlink} values of \emph{(2 +
number of sub directories)}.

Note that this is not a bug in S3QL. Including sub directories in
the \code{st\_nlink} value is a Unix convention, but by no means a
requirement. If an application blindly relies on this convention
being followed, then this is a bug in the application.

A prominent example are early versions of GNU find, which required
the \code{-{-}noleaf} option to work correctly on S3QL file systems. This
bug has already been fixed in recent find versions.

\item {} 
The \code{umount} and \code{fusermount -u} commands will \emph{not} block until all
data has been uploaded to the backend. (this is a FUSE limitation
that will hopefully be removed in the future, see \href{http://code.google.com/p/s3ql/issues/detail?id=159}{issue 159}). If you use
either command to unmount an S3QL file system, you have to take care
to explicitly wait for the \code{mount.s3ql} process to terminate before
you shut down or restart the system. Therefore it is generally not a
good idea to mount an S3QL file system in \code{/etc/fstab} (you should
use a dedicated init script instead).

\item {} 
S3QL relies on the backends not to run out of space. This is a given
for big storage providers like Amazon S3 or Google Storage, but you
may stumble upon this if you use your own server or smaller providers.

If there is no space left in the backend, attempts to write more
data into the S3QL file system will fail and the file system will be
in an inconsistent state and require a file system check (and you
should make sure to make space available in the backend before
running the check).

Unfortunately, there is no way to handle insufficient space in the
backend without leaving the file system inconsistent. Since
S3QL first writes data into the cache, it can no longer return an
error when it later turns out that the cache can not be committed to
the backend.

\end{itemize}


\chapter{Manpages}
\label{man/index:manpages}\label{man/index::doc}
The man pages are installed with S3QL on your system and can be viewed
with the \textbf{man} command. For reference, they are also included
here in the User's Guide.


\section{The \textbf{mkfs.s3ql} command}
\label{man/mkfs:the-mkfs-s3ql-command}\label{man/mkfs::doc}

\subsection{Synopsis}
\label{man/mkfs:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mkfs.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/mkfs:description}
The \textbf{mkfs.s3ql} command creates a new file system in the location
specified by \emph{storage url}. The storage url depends on the backend
that is used. The S3QL User's Guide should be consulted for a
description of the available backends.

Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask
you to enter an encryption password. This password will \emph{not} be read
from an authentication file specified with the \code{-{-}authfile}
option to prevent accidental creation of an encrypted file system.


\subsection{Options}
\label{man/mkfs:options}
The \textbf{mkfs.s3ql} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default:
\code{\textasciitilde{}/.s3ql)} (default: /home/nikratio/.s3ql)
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file
(default: \code{\textasciitilde{}/.s3ql/authinfo2)} (default:
/home/nikratio/.s3ql/authinfo2)
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to
get debug messages from all modules. This option can
be specified multiple times. (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}ssl]  
Always use SSL connections when connecting to remote
servers. For backends that allow only encrypted
connections, S3QL uses SSL automatically, even if this
option is not set. (default: False)
\item [-{-}version]  
just print program version and exit
\item [-L \textless{}name\textgreater{}]  
Filesystem label (default: )
\item [-{-}max-obj-size \textless{}size\textgreater{}]  
Maximum size of storage objects in KiB. Files bigger
than this will be spread over multiple objects in the
storage backend. Default: 10240 KiB.
\item [-{-}plain]  
Create unencrypted file system. (default: False)
\item [-{-}force]  
Overwrite any existing data. (default: False)
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/mkfs:exit-status}
\textbf{mkfs.s3ql} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/mkfs:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qladm} command}
\label{man/adm::doc}\label{man/adm:the-s3qladm-command}

\subsection{Synopsis}
\label{man/adm:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}

where \code{action} may be either of \textbf{passphrase},
\textbf{upgrade}, \textbf{delete} or \textbf{download-metadata}.


\subsection{Description}
\label{man/adm:description}
The \textbf{s3qladm} command performs various operations on \emph{unmounted} S3QL
file systems. The file system \emph{must not be mounted} when using
\textbf{s3qladm} or things will go wrong badly.

The storage url depends on the backend that is used. The S3QL User's
Guide should be consulted for a description of the available backends.


\subsection{Options}
\label{man/adm:options}
The \textbf{s3qladm} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get
debug messages from all modules. This option can be
specified multiple times. (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be rotated
when it reaches 1 MiB, and at most 5 old log files will
be kept. Specify \code{none} to disable logging. Default:
\code{none}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file (default:
\code{\textasciitilde{}/.s3ql/authinfo2)} (default:
/home/nikratio/.s3ql/authinfo2)
\item [-{-}ssl]  
Always use SSL connections when connecting to remote
servers. For backends that allow only encrypted
connections, S3QL uses SSL automatically, even if this
option is not set. (default: False)
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)}
(default: /home/nikratio/.s3ql)
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}

Hint: run \code{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional arguments
that the different actions take.


\subsection{Actions}
\label{man/adm:actions}
The following actions may be specified:
\begin{description}
\item[{passphrase}] \leavevmode
Changes the encryption passphrase of the file system.

\item[{upgrade}] \leavevmode
Upgrade the file system to the newest revision.

\item[{delete}] \leavevmode
Delete the file system with all the stored data.

\item[{download-metadata}] \leavevmode
Interactively download backups of the file system metadata.

\end{description}


\subsection{Exit Status}
\label{man/adm:exit-status}
\textbf{s3qladm} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/adm:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{mount.s3ql} command}
\label{man/mount::doc}\label{man/mount:the-mount-s3ql-command}

\subsection{Synopsis}
\label{man/mount:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mount point\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/mount:description}
The \textbf{mount.s3ql} command mounts the S3QL file system stored in \emph{storage
url} in the directory \emph{mount point}. The storage url depends on the
backend that is used. The S3QL User's Guide should be consulted for a
description of the available backends.


\subsection{Options}
\label{man/mount:options}
The \textbf{mount.s3ql} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be
rotated when it reaches 1 MiB, and at most 5 old log
files will be kept. Specify \code{none} to disable
logging. Default: \code{\textasciitilde{}/.s3ql/mount.log}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default:
\code{\textasciitilde{}/.s3ql)} (default: /home/nikratio/.s3ql)
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file
(default: \code{\textasciitilde{}/.s3ql/authinfo2)} (default:
/home/nikratio/.s3ql/authinfo2)
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to
get debug messages from all modules. This option can
be specified multiple times. (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}ssl]  
Always use SSL connections when connecting to remote
servers. For backends that allow only encrypted
connections, S3QL uses SSL automatically, even if this
option is not set. (default: False)
\item [-{-}version]  
just print program version and exit
\item [-{-}cachesize \textless{}size\textgreater{}]  
Cache size in KiB (default: 102400 (100 MiB)). Should
be at least 10 times the maximum object size of the
filesystem, otherwise an object may be retrieved and
written several times during a single write() or
read() operation. (default: 102400)
\item [-{-}max-cache-entries \textless{}num\textgreater{}]  
Maximum number of entries in cache (default: 768).
Each cache entry requires one file descriptor, so if
you increase this number you have to make sure that
your process file descriptor limit (as set with
\code{ulimit -n}) is high enough (at least the number of
cache entries + 100).
\item [-{-}allow-other]  
Normally, only the user who called \code{mount.s3ql} can
access the mount point. This user then also has full
access to it, independent of individual file
permissions. If the \code{-{-}allow-other} option is
specified, other users can access the mount point as
well and individual file permissions are taken into
account for all users. (default: False)
\item [-{-}allow-root]  
Like \code{-{-}allow-other}, but restrict access to the
mounting user and the root user. (default: False)
\item [-{-}fg]  
Do not daemonize, stay in foreground (default: False)
\item [-{-}single]  
Run in single threaded mode. If you don't understand
this, then you don't need it. (default: False)
\item [-{-}upstart]  
Stay in foreground and raise SIGSTOP once mountpoint
is up. (default: False)
\item [-{-}profile]  
Create profiling information. If you don't understand
this, then you don't need it. (default: False)
\item [-{-}compress \textless{}name\textgreater{}]  
Compression algorithm to use when storing new data.
Allowed values: \code{lzma}, \code{bzip2}, \code{zlib}, none.
(default: \code{lzma})
\item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}]  
Interval in seconds between complete metadata uploads.
Set to 0 to disable. Default: 24h. (default: 86400)
\item [-{-}threads \textless{}no\textgreater{}]  
Number of parallel upload threads to use (default:
auto). (default: None)
\item [-{-}nfs]  
Enable some optimizations for exporting the file
system over NFS. (default: False)
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/mount:exit-status}
\textbf{mount.s3ql} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/mount:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qlstat} command}
\label{man/stat:the-s3qlstat-command}\label{man/stat::doc}

\subsection{Synopsis}
\label{man/stat:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlstat }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/stat:description}
The \textbf{s3qlstat} command prints statistics about the S3QL file system mounted
at \code{mountpoint}.

\textbf{s3qlstat} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or
\code{-{-}allow-root}) the root user. This limitation might be
removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).


\subsection{Options}
\label{man/stat:options}
The \textbf{s3qlstat} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/stat:exit-status}
\textbf{s3qlstat} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/stat:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qlctrl} command}
\label{man/ctrl:the-s3qlctrl-command}\label{man/ctrl::doc}

\subsection{Synopsis}
\label{man/ctrl:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ ...}
\end{Verbatim}

where \code{action} may be either of \textbf{flushcache},
\textbf{upload-meta}, \textbf{cachesize} or
\textbf{log-metadata}.


\subsection{Description}
\label{man/ctrl:description}
The \textbf{s3qlctrl} command performs various actions on the S3QL file system mounted
in \code{mountpoint}.

\textbf{s3qlctrl} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or
\code{-{-}allow-root}) the root user. This limitation might be
removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).

The following actions may be specified:
\begin{description}
\item[{flushcache}] \leavevmode
Uploads all changed file data to the backend.

\item[{upload-meta}] \leavevmode
Upload metadata to the backend. All file system operations will
block while a snapshot of the metadata is prepared for upload.

\item[{cachesize}] \leavevmode
Changes the cache size of the file system. This action requires an
additional argument that specifies the new cache size in KiB, so the
complete command line is:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ cachesize }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}new-cache-size\PYGZgt{}}
\end{Verbatim}

\item[{log}] \leavevmode
Change the amount of information that is logged into
\code{\textasciitilde{}/.s3ql/mount.log} file. The complete syntax is:

\begin{Verbatim}[commandchars=\\\{\}]
s3qlctrl [options] log \textless{}mountpoint\textgreater{} \textless{}level\textgreater{} [\textless{}module\textgreater{} [\textless{}module\textgreater{} ...]]
\end{Verbatim}

here \code{level} is the desired new log level and may be either of
\emph{debug}, \emph{info} or \emph{warn}. One or more \code{module} may only be
specified with the \emph{debug} level and allow to restrict the debug
output to just the listed modules.

\end{description}


\subsection{Options}
\label{man/ctrl:options}
The \textbf{s3qlctrl} command also accepts the following options, no matter
what specific action is being invoked:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}

Hint: run \code{s3qlctrl \textless{}action\textgreater{} -{-}help} to get help on the additional arguments
that the different actions take.


\subsection{Exit Status}
\label{man/ctrl:exit-status}
\textbf{s3qlctrl} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/ctrl:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qlcp} command}
\label{man/cp:the-s3qlcp-command}\label{man/cp::doc}

\subsection{Synopsis}
\label{man/cp:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}source-dir\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}dest-dir\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/cp:description}
The \textbf{s3qlcp} command duplicates the directory tree \code{source-dir}
into \code{dest-dir} without physically copying the file contents.
Both source and destination must lie inside the same S3QL file system.

The replication will not take any additional space. Only if one of
directories is modified later on, the modified data will take
additional storage space.

\code{s3qlcp} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root})
the root user. This limitation might be removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).

Note that:
\begin{itemize}
\item {} 
After the replication, both source and target directory will still
be completely ordinary directories. You can regard \code{\textless{}src\textgreater{}} as a
snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the most common
usage of \code{s3qlcp} is to regularly duplicate the same source
directory, say \code{documents}, to different target directories. For a
e.g. monthly replication, the target directories would typically be
named something like \code{documents\_January} for the replication in
January, \code{documents\_February} for the replication in February etc.
In this case it is clear that the target directories should be
regarded as snapshots of the source directory.

\item {} 
Exactly the same effect could be achieved by an ordinary copy
program like \code{cp -a}. However, this procedure would be orders of
magnitude slower, because \code{cp} would have to read every file
completely (so that S3QL had to fetch all the data over the network
from the backend) before writing them into the destination folder.

\end{itemize}


\subsubsection{Snapshotting vs Hardlinking}
\label{man/cp:snapshotting-vs-hardlinking}
Snapshot support in S3QL is inspired by the hardlinking feature that
is offered by programs like \href{http://www.samba.org/rsync}{rsync} or
\href{http://savannah.nongnu.org/projects/storebackup}{storeBackup}.
These programs can create a hardlink instead of copying a file if an
identical file already exists in the backup. However, using hardlinks
has two large disadvantages:
\begin{itemize}
\item {} 
backups and restores always have to be made with a special program
that takes care of the hardlinking. The backup must not be touched
by any other programs (they may make changes that inadvertently
affect other hardlinked files)

\item {} 
special care needs to be taken to handle files which are already
hardlinked (the restore program needs to know that the hardlink was
not just introduced by the backup program to safe space)

\end{itemize}

S3QL snapshots do not have these problems, and they can be used with
any backup program.


\subsection{Options}
\label{man/cp:options}
The \textbf{s3qlcp} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/cp:exit-status}
\textbf{s3qlcp} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/cp:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qlrm} command}
\label{man/rm::doc}\label{man/rm:the-s3qlrm-command}

\subsection{Synopsis}
\label{man/rm:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlrm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}directory\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/rm:description}
The \textbf{s3qlrm} command recursively deletes files and directories on an
S3QL file system. Although \textbf{s3qlrm} is faster than using e.g.
\textbf{rm -r{}`}, the main reason for its existence is that it allows
you to delete immutable trees (which can be created with
\textbf{s3qllock}) as well.

Be warned that there is no additional confirmation. The directory will
be removed entirely and immediately.

\textbf{s3qlrm} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or
\code{-{-}allow-root}) the root user. This limitation might be
removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).


\subsection{Options}
\label{man/rm:options}
The \textbf{s3qlrm} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/rm:exit-status}
\textbf{s3qlrm} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/rm:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qllock} command}
\label{man/lock:the-s3qllock-command}\label{man/lock::doc}

\subsection{Synopsis}
\label{man/lock:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qllock }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}directory\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/lock:description}
The \textbf{s3qllock} command makes a directory tree in an S3QL file
system immutable. Immutable trees can no longer be changed in any way
whatsoever. You can not add new files or directories and you can not
change or delete existing files and directories. The only way to get
rid of an immutable tree is to use the \textbf{s3qlrm} command.

\textbf{s3qllock} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or
\code{-{-}allow-root}) the root user. This limitation might be
removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).


\subsection{Rationale}
\label{man/lock:rationale}
Immutability is a feature designed for backups. Traditionally, backups
have been made on external tape drives. Once a backup was made, the
tape drive was removed and locked somewhere in a shelf. This has the
great advantage that the contents of the backup are now permanently
fixed. Nothing (short of physical destruction) can change or delete
files in the backup.

In contrast, when backing up into an online storage system like S3QL,
all backups are available every time the file system is mounted.
Nothing prevents a file in an old backup from being changed again
later on. In the worst case, this may make your entire backup system
worthless. Imagine that your system gets infected by a nasty virus
that simply deletes all files it can find -- if the virus is active
while the backup file system is mounted, the virus will destroy all
your old backups as well!

Even if the possibility of a malicious virus or trojan horse is
excluded, being able to change a backup after it has been made is
generally not a good idea. A common S3QL use case is to keep the file
system mounted at all times and periodically create backups with
\textbf{rsync -a}. This allows every user to recover her files from a
backup without having to call the system administrator. However, this
also allows every user to accidentally change or delete files \emph{in} one
of the old backups.

Making a backup immutable protects you against all these problems.
Unless you happen to run into a virus that was specifically programmed
to attack S3QL file systems, backups can be neither deleted nor
changed after they have been made immutable.


\subsection{Options}
\label{man/lock:options}
The \textbf{s3qllock} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/lock:exit-status}
\textbf{s3qllock} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/lock:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{umount.s3ql} command}
\label{man/umount::doc}\label{man/umount:the-umount-s3ql-command}

\subsection{Synopsis}
\label{man/umount:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{umount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mount point\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/umount:description}
The \textbf{umount.s3ql} command unmounts the S3QL file system mounted in the
directory \emph{mount point} and blocks until all data has been uploaded to
the storage backend.

Only the user who mounted the file system with \textbf{mount.s3ql}
is able to unmount it with \textbf{umount.s3ql}. If you are root and want to
unmount an S3QL file system mounted by an ordinary user, you have to
use the \textbf{fusermount -u} or \textbf{umount} command instead.
Note that these commands do not block until all data has been
uploaded, so if you use them instead of \textbf{umount.s3ql} then
you should manually wait for the \textbf{mount.s3ql} process to
terminate before shutting down the system.


\subsection{Options}
\label{man/umount:options}
The \textbf{umount.s3ql} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}version]  
just print program version and exit
\item [-{-}lazy, -z]  
Lazy umount. Detaches the file system immediately, even if
there are still open files. The data will be uploaded in the
background once all open files have been closed. (default:
False)
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/umount:exit-status}
\textbf{umount.s3ql} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/umount:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{fsck.s3ql} command}
\label{man/fsck::doc}\label{man/fsck:the-fsck-s3ql-command}

\subsection{Synopsis}
\label{man/fsck:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{fsck.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/fsck:description}
The \textbf{mkfs.s3ql} command checks the new file system in the location
specified by \emph{storage url} for errors and attempts to repair any
problems. The storage url depends on the backend that is used. The
S3QL User's Guide should be consulted for a description of the
available backends.


\subsection{Options}
\label{man/fsck:options}
The \textbf{mkfs.s3ql} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be rotated
when it reaches 1 MiB, and at most 5 old log files will
be kept. Specify \code{none} to disable logging. Default:
\code{\textasciitilde{}/.s3ql/fsck.log}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)}
(default: /home/nikratio/.s3ql)
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file (default:
\code{\textasciitilde{}/.s3ql/authinfo2)} (default:
/home/nikratio/.s3ql/authinfo2)
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get
debug messages from all modules. This option can be
specified multiple times. (default: None)
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}ssl]  
Always use SSL connections when connecting to remote
servers. For backends that allow only encrypted
connections, S3QL uses SSL automatically, even if this
option is not set. (default: False)
\item [-{-}version]  
just print program version and exit
\item [-{-}batch]  
If user input is required, exit without prompting.
(default: False)
\item [-{-}force]  
Force checking even if file system is marked clean.
(default: False)
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/fsck:exit-status}
\textbf{mkfs.s3ql} returns exit code 0 if the operation succeeded and 1 if some
error occurred.


\subsection{See Also}
\label{man/fsck:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, common locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{pcp} command}
\label{man/pcp:the-pcp-command}\label{man/pcp::doc}

\subsection{Synopsis}
\label{man/pcp:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{pcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}source\PYGZgt{}}\PYG{l}{ }\PYG{g+ge}{[\PYGZlt{}source\PYGZgt{} ...]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}destination\PYGZgt{}}
\end{Verbatim}


\subsection{Description}
\label{man/pcp:description}
The \textbf{pcp} command is a is a wrapper that starts several
\textbf{sync} processes to copy directory trees in parallel. This is
allows much better copying performance on file system that have
relatively high latency when retrieving individual files like S3QL.

\textbf{Note}: Using this program only improves performance when copying
\emph{from} an S3QL file system. When copying \emph{to} an S3QL file system,
using \textbf{pcp} is more likely to \emph{decrease} performance.


\subsection{Options}
\label{man/pcp:options}
The \textbf{pcp} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}version]  
just print program version and exit
\item [-a]  
Pass -aHAX option to rsync. (default: False)
\item [-{-}processes \textless{}no\textgreater{}]  
Number of rsync processes to use (default: 10).
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/pcp:exit-status}
\textbf{pcp} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/pcp:see-also}
\textbf{pcp} is shipped as part of S3QL, \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.


\section{The \textbf{expire\_backups} command}
\label{man/expire_backups::doc}\label{man/expire_backups:the-expire-backups-command}

\subsection{Synopsis}
\label{man/expire_backups:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{expire\PYGZus{}backups }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}age\PYGZgt{}}\PYG{l}{ }\PYG{g+ge}{[\PYGZlt{}age\PYGZgt{} ...]}
\end{Verbatim}


\subsection{Description}
\label{man/expire_backups:description}
The \textbf{expire\_backups} command intelligently remove old backups that are no
longer needed.

To define what backups you want to keep for how long, you define a
number of \emph{age ranges}. \textbf{expire\_backups} ensures that you
will have at least one backup in each age range at all times. It will
keep exactly as many backups as are required for that and delete any
backups that become redundant.

Age ranges are specified by giving a list of range boundaries in terms
of backup cycles. Every time you create a new backup, the existing
backups age by one cycle.

Example: when \textbf{expire\_backups} is called with the age range
definition \code{1 3 7 14 31}, it will guarantee that you always have the
following backups available:
\begin{enumerate}
\item {} 
A backup that is 0 to 1 cycles old (i.e, the most recent backup)

\item {} 
A backup that is 1 to 3 cycles old

\item {} 
A backup that is 3 to 7 cycles old

\item {} 
A backup that is 7 to 14 cycles old

\item {} 
A backup that is 14 to 31 cycles old

\end{enumerate}

\begin{notice}{note}{Note:}
If you do backups in fixed intervals, then one cycle will be
equivalent to the backup interval. The advantage of specifying the
age ranges in terms of backup cycles rather than days or weeks is
that it allows you to gracefully handle irregular backup intervals.
Imagine that for some reason you do not turn on your computer for
one month. Now all your backups are at least a month old, and if you
had specified the above backup strategy in terms of absolute ages,
they would all be deleted! Specifying age ranges in terms of backup
cycles avoids these sort of problems.
\end{notice}

\textbf{expire\_backups} usage is simple. It requires backups to have
names of the forms \code{year-month-day\_hour:minute:seconds}
(\code{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current
directory. So for the above backup strategy, the correct invocation
would be:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{expire\PYGZus{}backups.py 1 3 7 14 31}
\end{Verbatim}

When storing your backups on an S3QL file system, you probably want to
specify the \code{-{-}use-s3qlrm} option as well. This tells
\textbf{expire\_backups} to use the {\hyperref[special:s3qlrm]{\emph{s3qlrm}}} command to
delete directories.

\textbf{expire\_backups} uses a ``state file'' to keep track which
backups are how many cycles old (since this cannot be inferred from
the dates contained in the directory names). The standard name for
this state file is \code{.expire\_backups.dat}. If this file gets
damaged or deleted, \textbf{expire\_backups} no longer knows the ages
of the backups and refuses to work. In this case you can use the
\code{-{-}reconstruct-state} option to try to reconstruct the state
from the backup dates. However, the accuracy of this reconstruction
depends strongly on how rigorous you have been with making backups (it
is only completely correct if the time between subsequent backups has
always been exactly the same), so it's generally a good idea not to
tamper with the state file.


\subsection{Options}
\label{man/expire_backups:options}
The \textbf{expire\_backups} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}quiet]  
be really quiet (default: False)
\item [-{-}debug]  
activate debugging output (default: None)
\item [-{-}version]  
just print program version and exit
\item [-{-}state \textless{}file\textgreater{}]  
File to save state information in (default:
''.expire\_backups.dat'')
\item [-n]  
Dry run. Just show which backups would be deleted.
(default: False)
\item [-{-}reconstruct-state]  
Try to reconstruct a missing state file from backup
dates. (default: False)
\item [-{-}use-s3qlrm]  
Use \code{s3qlrm} command to delete backups. (default:
False)
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/expire_backups:exit-status}
\textbf{expire\_backups} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/expire_backups:see-also}
\textbf{expire\_backups} is shipped as part of S3QL, \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.


\chapter{Further Resources / Getting Help}
\label{resources::doc}\label{resources:further-resources-getting-help}\label{resources:resources}
If you have questions or problems with S3QL that you weren't able to
resolve with this manual, you might want to consider the following other resources:
\begin{itemize}
\item {} 
The \href{http://code.google.com/p/s3ql/w/list}{S3QL Wiki}

\item {} 
The \href{http://code.google.com/p/s3ql/wiki/FAQ}{S3QL FAQ}

\item {} 
The \href{http://groups.google.com/group/s3ql}{S3QL Mailing List}. You
can subscribe by sending a mail to
\href{mailto:s3ql+subscribe@googlegroups.com}{s3ql+subscribe@googlegroups.com}.

\end{itemize}

Please report any bugs you may encounter in the \href{http://code.google.com/p/s3ql/issues/list}{Issue Tracker}.


\chapter{Implementation Details}
\label{impl_details:implementation-details}\label{impl_details::doc}\label{impl_details:impl-details}
This section provides some background information on how S3QL works
internally. Reading this section is not necessary to use S3QL.


\section{Metadata Storage}
\label{impl_details:metadata-storage}
Like most unix filesystems, S3QL has a concept of inodes.

The contents of directory inodes (aka the names and inodes of the
files and sub directories contained in a directory) are stored
directly in a {[}\href{http://www.sqlite.org}{http://www.sqlite.org} SQLite{]} database. This database
is stored in a special S3 object that is downloaded when the file
system is mounted and uploaded periodically in the background and when
the file system is unmounted. This has two implications:
\begin{enumerate}
\item {} 
The entire file system tree can be read from the database. Fetching
or storing S3 objects is only required to access the contents of
files (or, more precisely, inodes). This makes most file system
operations very fast because no data has to be send over the
network.

\item {} 
An S3QL filesystem can only be mounted at one position at a time,
otherwise changes made in one mountpoint will invariably be
overwritten when the second mount point is unmounted.

\end{enumerate}

Sockets, FIFOs and character devices do not need any additional
storage, all information about them is contained in the database.


\section{Data Storage}
\label{impl_details:data-storage}
The contents of file inodes are split into individual blocks. The
maximum size of a block is specified when the file system is created
and cannot be changed afterwards. Every block is stored as an
individual object in the backend, and the mapping from inodes to
blocks and from blocks to objects is stored in the database.

While the file system is mounted, blocks are cached locally.

Blocks can also be compressed and encrypted before they are stored in
S3.

If some files have blocks with identical contents, the blocks will be
stored in the same backend object (i.e., the data is only stored
once).


\section{Data De-Duplication}
\label{impl_details:data-de-duplication}
Instead of uploading every block, S3QL first computes a checksum (a
SHA256 hash, for those who are interested) to check if an identical
blocks has already been stored in an backend object. If that is the
case, the new block will be linked to the existing object instead of
being uploaded.

This procedure is invisible for the user and the contents of the block
can still be changed. If several blocks share a backend object and one
of the blocks is changed, the changed block is automatically stored in
a new object (so that the contents of the other block remain
unchanged).


\section{Caching}
\label{impl_details:caching}
When an application tries to read or write from a file, S3QL
determines the block that contains the required part of the file and
retrieves it from the backend or creates it if it does not yet exist.
The block is then held in the cache directory. It is committed to S3
when it has not been accessed for more than 10 seconds. Blocks are
removed from the cache only when the maximum cache size is reached.

When the file system is unmounted, all modified blocks are committed
to the backend and the cache is cleaned.


\section{Eventual Consistency Handling}
\label{impl_details:eventual-consistency-handling}
S3QL has to take into account that changes in objects do not propagate
immediately in all backends. For example, when an Amazon S3 object is
uploaded and immediately downloaded again, the downloaded data might
not yet reflect the changes done in the upload (see also
\href{http://developer.amazonwebservices.com/connect/message.jspa?messageID=38538}{http://developer.amazonwebservices.com/connect/message.jspa?messageID=38538})

For the data blocks this is not a problem because a data blocks always
get a new object ID when they are updated.

For the metadata however, S3QL has to make sure that it always
downloads the most recent copy of the database when mounting the file
system.

To that end, metadata versions are numbered, and the most recent
version number is stored as part of the object id of a very small
``marker'' object. When S3QL has downloaded the metadata it checks the
version number against the marker object and, if the two do not agree,
waits for the most recent metadata to become available. Once the
current metadata is available, the version number is increased and the
marker object updated.


\section{Encryption}
\label{impl_details:encryption}
When the file system is created, \code{mkfs.s3ql} generates a 256 bit
master key by reading from \code{/dev/random}. The master key is encrypted
with the passphrase that is entered by the user, and then stored with
the rest of the file system data. Since the passphrase is only used to
access the master key (which is used to encrypt the actual file system
data), the passphrase can easily be changed.

Data is encrypted with a new session key for each object and each
upload. The session key is generated by appending a nonce to the
master key and then calculating the SHA256 hash. The nonce is
generated by concatenating the object id and the current UTC time as a
32 bit float. The precision of the time is given by the
{[}\href{http://docs.python.org/library/time.html\#time.time}{http://docs.python.org/library/time.html\#time.time} Python \code{time()}
function{]} and usually at least 1 millisecond. The SHA256
implementation is included in the Python standard library.

Once the session key has been calculated, a SHA256 HMAC is calculated
over the data that is to be uploaded. Afterwards, the data is
compressed with the LZMA, {[}\href{http://en.wikipedia.org/wiki/Bz2}{http://en.wikipedia.org/wiki/Bz2} Bz2
algorithm{]} or LZ and the HMAC inserted at the beginning. Both HMAC and
compressed data are then encrypted using 256 bit AES in CTR mode. The
AES-CTR implementation is provided by the {[}\href{http://cryptopp.com/}{http://cryptopp.com/}
Crypto++{]} library. Finally, the nonce is inserted in front of the
encrypted data and HMAC, and the packet is send to the backend as a
new S3 object.



\renewcommand{\indexname}{Index}
\printindex
\end{document}
