| Current File : //usr/lib64/python3.6/site-packages/borg/__pycache__/archiver.cpython-36.pyc |
3
up�d/ � @ s� d dl Z d dlZ�y2d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl Z d dl
Z
d dl Z d dl Z d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZ d dlmZ d dlmZm Z d dl m Z ddl m Z m Z e � Z d dl!Z!ddl"m#Z# dd l"m$Z$ dd
l"m%Z% dd l&m'Z' dd l(m)Z)m*Z*m+Z+m,Z,m-Z- dd
l(m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4m5Z5 ddl6T ddl7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? ddl@mAZA ddl$mBZBmCZCmDZDmEZE ddl$mFZFmGZGmHZH ddl$mIZImJZJmKZKmLZLmMZM ddl$mNZNmOZOmPZPmQZQmRZRmSZS ddl$mTZTmUZUmVZV ddl$mWZWmXZXmYZYmZZZ ddl$m[Z[m\Z\m]Z]m^Z^ ddl$m_Z_m`Z`maZa ddl$mbZb dd l$mcZc dd l$mdZd dd l$meZe dd l$mfZf dd l$mgZgmhZh dd!l$miZimjZjmkZkmlZlmmZm dd"l$mnZn dd#l$moZompZpmqZqmrZr dd$l$msZs dd%l$mtZt dd&l$muZumvZv dd'l$mwZw dd(l$mxZx dd)l$myZymzZz dd*l$m{Z{ dd+l$m|Z| dd,l$m}Z}m~Z~ dd-l$mZm�Z� dd.l$m�Z� dd/l�m�Z� dd0l�m�Z�m�Z�m�Z�m�Z� dd1l�m�Z� dd2l�m�Z�m�Z� dd3l�m�Z�m�Z�m�Z� dd4l�m�Z�m�Z�m�Z� dd5l�m�Z�m�Z�m�Z�m�Z�m�Z� dd6l�m�Z� dd7l�m�Z�m�Z� W n( e�k
�rl ej�� e j�d8� Y nX eDd8k�s�t�d9��d:Z�d;Z�d<d=� Z�dQd@dA�Z�dBdC� Z�dDdE� Z�dFdG� Z�G dHdI� dI�Z�dJdK� Z�dLdM� Z�dNdO� Z�e�dPk�r�e�� dS )R� N)� unhexlify�hexlify)�contextmanager)�datetime� timedelta)� zip_longest� )�
create_logger�
setup_logging)� __version__)�helpers)� shellpattern)�crc32)�Archive�ArchiveChecker�ArchiveRecreater�
Statistics�
is_special)� BackupError�
BackupOSError� backup_io�has_link)�Cache�
assert_secure�SecurityManager)�*)�CompressionSpec)� key_creator�key_argument_names�tam_required_file� tam_required�RepoKey�
PassphraseKey)�
KeyManager)� EXIT_SUCCESS� EXIT_WARNING�
EXIT_ERROR�EXIT_SIGNAL_BASE)�Error�NoManifestError�set_ec)�positive_int_validator�location_validator�archivename_validator�
ChunkerParams�Location)�
PrefixSpec�GlobSpec� CommentSpec�
SortBySpec�HUMAN_SORT_KEYS�FilesCacheMode)�
BaseFormatter�
ItemFormatter�ArchiveFormatter)�format_timedelta�format_file_size�parse_file_size�format_archive)� safe_encode�remove_surrogates�
bin_to_hex�prepare_dump_dict)�interval� prune_within� prune_split)� timestamp)�
get_cache_dir)�Manifest)� hardlinkable)�
StableDict)� check_python�check_extension_modules)�
dir_is_tagged�is_slow_msgpack�is_supported_msgpack�yes�sysinfo)� log_multi)�signal_handler�raising_signal_handler�SigHup�SigTerm)�ErrorIgnoringTextIOWrapper)�ProgressIndicatorPercent)�basic_json_data�
json_print)�replace_placeholders)�ChunkIteratorFileWrapper)�popen_with_error_handling�prepare_subprocess_env)� dash_open)�umount)�msgpack�msgpack_fallback)�uid2user� gid2group)� safe_decode)�rst_to_terminal)�ArgparsePatternAction�ArgparseExcludeFileAction�ArgparsePatternFileAction�parse_exclude_pattern)�PatternMatcher)�Item� ArchiveItem)� get_flags�get_process_id�SyncFile)�RepositoryServer�RemoteRepository�cache_if_remote)�
Repository�LIST_SCAN_LIMIT�TAG_PUT�
TAG_DELETE�
TAG_COMMIT)�selftest)�AtticRepositoryUpgrader�BorgRepositoryUpgrader� zZEXIT_ERROR is not 2, as expected - fix assert AND exception handler right above this line.zN Original size Compressed size Deduplicated sizezCUsing a pure-python msgpack! This will result in lower performance.c s< t |t�rt� |�S t |ttf�r8t� fdd�|D ��S |S )zSIf bool is passed, return it. If str is passed, retrieve named attribute from args.c 3 s | ]}t � |�V qd S )N)�getattr)�.0�item)�args� � /usr/lib64/python3.6/archiver.py� <genexpr>h s zargument.<locals>.<genexpr>)�
isinstance�strr{ �list�tuple�any)r~ Z str_or_boolr )r~ r� �argumentc s
r� FTc
sz � r4�s�r4�dkr t d��t��tk rVt d��n"�dk rLt dt�� ���rVtj�|� � ��������f dd�} | S )u�
Method decorator for subcommand-handling methods: do_XYZ(self, args, repository, …)
If a parameter (where allowed) is a str the attribute named of args is used instead.
:param fake: (str or bool) use None instead of repository, don't do anything else
:param create: create repository
:param lock: lock repository
:param exclusive: (str or bool) lock repository exclusively (for writing)
:param manifest: load manifest and key, pass them as keyword arguments
:param cache: open cache, pass it as keyword argument (implies manifest)
:param secure: do assert_secure after loading manifest
:param compatibility: mandatory if not create and (manifest or cache), specifies mandatory feature categories to check
Nz=with_repository decorator used without compatibility argumentzFwith_repository decorator compatibility argument must be of type tuplezFwith_repository called with compatibility argument but would not checkc s, t j� ���������� � f
dd��}|S )Nc
s� t |d� �}|j}t |dd�}t |dd �}t |dd�}t|���A rZ�| |fdd i|��S |jdkr�t|j� �t|��| j||||d�}n"t|j�t|��| j||||d �}|�� �s���rt j
|��\|d
< |d < d |kr�|j j |d _ � �rt
||d
| j� ��rlt||d |d
t |d
d�t |dd�t |dd�| jt |dt�d��} �| |f|| d�|��S Q R X n�| |fd|i|��S W d Q R X d S )N�lock� append_onlyF�
storage_quota�make_parent_dirs�
repositoryZssh)�create� exclusive� lock_waitr� r� r� r~ )r� r� r� r� r� r� r� �manifest�key� compression� cache_files� ignore_inode�progress�files_cache_mode)�do_filesr� r� r� �
cache_mode)r� �cache)r{ �locationr� �protorp Z omit_archiver� rr �pathrF �loadr� Z
compressorr r ZFILES_CACHE_MODE_DISABLED)
�selfr~ �kwargsr� r� r� r� r� r� Zcache_)
�_lockr� �
compatibilityr� r� �fake� invert_faker� �method�securer r� �wrapper� s:
z3with_repository.<locals>.decorator.<locals>.wrapper)� functools�wraps)r� r� ) r� r� r� r� r� r� r� r� r� )r� r� � decorator� s (!z"with_repository.<locals>.decorator)�AssertionError�typer� �reprrF �NO_OPERATION_CHECK)
r� r� r� r� r� r� r� r� r� r� r ) r� r� r� r� r� r� r� r� r� r� �with_repositoryl s
#r� c s t j� �� fdd��}|S )Nc
sh t ||||jjt|dd�t|dd�t|dd�t|dd�|jd�|j|jd� }� | |f||||d�|��S ) N�
numeric_ownerF�
nobsdflags�noacls�noxattrsr� )r� r� r� r� r� �consider_part_files�log_json)r� r� r� �archive)r r� r� r{ �getr� r� )r� r~ r� r� r� r� r� )r� r r� r� � s
z with_archive.<locals>.wrapper)r� r� )r� r� r )r� r� � with_archive� s
r� c C s& t | �}|t d�k r"tjd| ��|S )N�10Mz3quota is too small (%s). At least 10M are required.)r; �argparseZArgumentTypeError)r� Zparsedr r r� �parse_storage_quota� s r� c C s2 x$dD ] }t | |d �}|d k r|S qW td��d S )N�func�
fallback_func�fallback2_funcz"expected func attributes not found)r� r� r� )r{ � Exception)r~ �namer� r r r� �get_func� s
r� c @ s* e Zd Zd�dd�Zdd� Zdd� Zdd � Zed
d � �Zed d
� �Z dd� Z
e dddd�dd� �Z e ddd�dd� �Z
e ejjfd�dd� �Zdd � Ze ddddd �d d � �Ze ddddd �d d!� �Ze dd"�d#d$� �Zd%d&� Ze d'dejjfd(�d�d)d*��Zd�d+d,�Zed-d.� �Ze ejjfd�e d/d0� ��Z e ejjfd�e d1d2� ��Z d3d4� Z e ejjfd�e d5d6� ��Z e ddejjfd7�e d8d9� ��Z!e ddd�d:d;� �Z"d<d=� Z#d>d?� Z$d@dA� Z%e ejjfd�dBdC� �Z&dDdE� Z'e ejjfd�dFdG� �Z(dHdI� Z)dJdK� Z*e dejjfdL�dMdN� �Z+dOdP� Z,dQdR� Z-e dejj.fdS�dTdU� �Z/e d�ddddY�d�dZd[��Z0e ddejjfd\�d]d^� �Z1e ddd_�d`da� �Z2e ddd�dbdc� �Z3ddde� Z4e ej5d�dfdg� �Z6e ej5d�dhdi� �Z7e ej5d�djdk� �Z8e dd"�dldm� �Z9e dd"�dndo� �Z:e dd"�dpdq� �Z;e ddd_�drds� �Z<e ddd_�dtdu� �Z=e dddej5dv�dwdx� �Z>e ddd_�dydz� �Z?d{d|� Z@e ddd}�d~d� �ZAeBjC� ZDeEjFd��eDd�< eEjFd��eDd�< eEjFd��eDd�< d�d�� ZGd�d�� ZHeHZId�d�� ZJG d�d�� d��ZKd�d�� ZLd�d�� ZMd�d�d��ZNd�d�� ZOd�d�� ZPd�d�� ZQd�d�� ZRdS )��ArchiverNc C s t | _|| _|| _d S )N)r$ � exit_coder� �prog)r� r� r� r r r� �__init__� s zArchiver.__init__c G s$ |r || p|}t | _tj|� d S )N)r& r� �logger�error)r� �msgr~ r r r� � print_error� s zArchiver.print_errorc G s$ |r || p|}t | _tj|� d S )N)r% r� r� �warning)r� r� r~ r r r� �
print_warning� s zArchiver.print_warningc C s^ | j rZ| jd ks|| jkrZ| jrBttjd|t|�d��tjd� nt j
d�j d|t|�� d S )NZ file_status)r� �statusr� )�filezborg.output.listz%1s %s) � output_list�
output_filterr� �print�json�dumpsr> �sys�stderr�logging� getLogger�info)r� r� r� r r r� �print_file_status� s zArchiver.print_file_statusc
C s� t � }d }}d }}x�|| s^t| |�}||krJ|| oHt||�|kS t|�}t|�}d}|| s�t||�}||kr�|| o�t| |�|kS t|�}t|�}d}t|| || �} |||| � |||| � kr�dS || 7 }|| 7 }qW dS )zSCompare two chunk iterators (like returned by :meth:`.DownloadPipeline.fetch_many`)r FN)�object�next�
memoryview�len�min)
�chunks1�chunks2�endZalenZaiZblenZbi�a�bZslicelenr r r� �compare_chunk_contents� s.
z Archiver.compare_chunk_contentsc C s t � }|j| � |j|� |S )N)ri � add_inclexclZadd_includepaths)Zinclexcl_patternsZ
include_paths�matcherr r r� �
build_matcher s
zArchiver.build_matcherc C s t |j|j|j|jd�j� tS )z@Start in server mode. This command is usually not used manually.)�restrict_to_paths�restrict_to_repositoriesr� r� )ro r� r� r� r� �server$ )r� r~ r r r� �do_serve s zArchiver.do_serveTF)r� r� r� c C s� |j j� }tjd| � yt||�}W n ttfk
rF |j� tS X t ||�}||_
|j � |j � t
|||dd�� W dQ R X |jr�t|�}t|d�j� tjdtj|�� |jdkr�tjd� | jS ) z Initialize an empty repositoryz Initializing repository at "%s"F)Zwarn_if_unencryptedN�wa�
By default repositories initialized with this version will produce security
errors if written to with an older version (up to and including Borg 1.0.8).
If you want to use these older versions, you can disable the check by running:
borg upgrade --disable-tam %s
See https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability for details about the security implications.� plaintexta)
IMPORTANT: you will need both KEY AND PASSPHRASE to access this repo!
If you used a repokey mode, the key is stored in the repo, but you should back it up separately.
Use "borg key export" to export the key, optionally in printable format.
Write down the passphrase. Store both at safe place(s).
)r� �canonical_pathr� r� r �EOFError�KeyboardInterrupt�destroyr% rF r� �write�commitr r r �open�closer� �shlex�quote�NAMEr� )r� r~ r� r� r� r� �tam_filer r r� �do_init s.
zArchiver.do_init)r� r� c C s� |j r"d}t|dddddd�s"tS |jrVt|j|j|j|jdk |j f�rV| j
d � tS |j sr|j |j |j
d
�srtS |jdk r�|jd |_ |j r�t� j ||j |jj|j|j|jp�d |j |j|j
d
� r�tS tS )z Check repository consistencyz�This is a potentially dangerous function.
check --repair might lead to data loss (for kinds of corruption it is not
capable of dealing with). BE VERY CAREFUL!
Type 'YES' if you understand this and want to continue: z Aborting.zInvalid answer, aborting.�YESFZ!BORG_CHECK_I_KNOW_WHAT_I_AM_DOING)� false_msg� invalid_msg�truish�retry�env_var_overrideNzT--repository-only contradicts --first, --last, --prefix and --verify-data arguments.)�repair�
save_spacer �ts)r� r� �first�last�sort_by�glob� verify_datar )r� )r� rN r& � repo_onlyr� r r r �prefix�
glob_archivesr� �
archives_only�checkr r% r r� r� r r$ )r� r~ r� r� r r r� �do_checkG s*
zArchiver.do_check)r� c C sF t |d�std� tS |j� tjd� t |d�rBtjd|j� � tS )z%Change repository key file passphrase�change_passphrasez?This repository is not encrypted, cannot change the passphrase.z Key updated�find_keyzKey location: %s)�hasattrr� r& r
r� r� r r$ )r� r~ r� r� r� r r r� �do_change_passphrased s
z Archiver.do_change_passphrasec C s t jd� | j|�S )Nzq"borg change-passphrase" is deprecated and will be removed in Borg 1.2.
Use "borg key change-passphrase" instead.)r� r� r )r� r~ r r r� � do_change_passphrase_deprecatedq s
z(Archiver.do_change_passphrase_deprecated)r� r� r� r� c C st t |�}|j� |jr$|j|j� nLy$|jr:|j|j� n |j|j� W n& tk
rn | j dj
|j�� t S X t S )z$Export the repository key for backupz$'{}' must be a file, not a directory)
r# Z load_keyblob�paperZexport_paperkeyr� �qrZ export_qr�export�IsADirectoryErrorr� �formatr& r$ )r� r~ r� �managerr r r� �
do_key_exportv s zArchiver.do_key_exportc C s~ t |�}|jr.|jr"| jd� tS |j|� nL|jsB| jd� tS |jdkrptjj|j� rp| jd|j � tS |j|� t S )z%Import the repository key from backupz.with --paper import from file is not supportedz&input file to import key from expected�-zinput file does not exist: )
r# r r� r� r& Zimport_paperkey�os�existsZimport_keyfiler$ )r� r~ r� r r r r� �
do_key_import� s
zArchiver.do_key_import)r� c C sd |j tj�}tj||�}t|�}||_|j|_|j |_ |j
|_
|j |_ |j |_ |j
� tjd� tS )z Migrate passphrase -> repokeyz Key updated)r� rF � MANIFEST_IDr" Zdetectr! �target�idZ
repository_idZenc_keyZ enc_hmac_keyZid_keyZ
chunk_seedr
r� r� r$ )r� r~ r� Z
manifest_dataZkey_oldZkey_newr r r� �do_migrate_to_repokey� s
z Archiver.do_migrate_to_repokeyc s& � fdd�}t dd� �}dtjkr,d d!g}nd"d#d$d%d&d'g}x�|D ]�\}}}}||j|||�� } ||jj� | �\}
} } }
W dQ R X || d }t|�}|r�dnd}d}t|d|||
||||
f � t|d ||| |||| f � t|d ||| |||| f � t|d |||
||||
f � qBW d S )(z4Benchmark Create, Read, Update, Delete for archives.c
s� | d }d}t j� }� j� jd|d|d |g��}t j� }|| }|dksNt�� j� jd||d |g��}� j� jd|d g��} || ko�dkn s�t�t j� }� j� jd||d |g��}t j� }|| }
� j� jd|d g��} || ko�dkn �st�t j� }� j� jd
d |d g��}t j� }|| } |dk�sHt�t j� }� j� jd|d g��}t j� }|| } |dk�s�t�||
| | fS ) Nz::borg-benchmark-crudz--compression=noner� z--files-cache=disabled�1r �2�delete�3�extractz --dry-run)�time� monotonic� do_create�
parse_argsr� � do_delete�
do_extract)
�repor� r� r� Zt_start�rcZt_end� dt_createZrc1Zrc2� dt_update�
dt_extract� dt_delete)r� r r� �measurement_run� s6 z3Archiver.do_benchmark_crud.<locals>.measurement_runc s s� z~t jj| d�} t j| � xZt|�D ]N}t jj| d| �}|sFd| nt j|�}t|dd��}|j|� W d Q R X q$W | V W d tj | � X d S )Nzborg-test-datazfile_%d� T)Zbinary)
r r� �join�makedirs�range�urandomrn r� �shutilZrmtree)r� �count�size�random�iZfname�data�fdr r r� �
test_files� s
z.Archiver.do_benchmark_crud.<locals>.test_filesZ_BORG_BENCHMARK_CRUD_TEST�Z-TESTr F�R-TESTT�Z-BIG�
� ��R-BIG�Z-MEDIUM�� �@B �R-MEDIUM�Z-SMALL�' �R-SMALLNg ��.Ar; zall-zeroz-%s-%-10s %9.2f MB/s (%d * %s %s files: %.2fs)�C�R�U�Dr )r@ r r F)rA r r T)rB rC rD F)rE rC rD T)rF rG rH F)rI rG rH T)rJ rK rK F)rL rK rK T)r r �environr� r� r� r: r� )r� r~ r2 r? Ztestsr� r9 r: r; r� r. r/ r0 r1 Z
total_size_MBZfile_size_formattedZcontentZfmtr )r� r� �do_benchmark_crud� s. "
" "zArchiver.do_benchmark_crud�dry_run)r� r� r� c
s t dd���j� j� � ����fdd�}� j�_� j�_� j�_� j�tj� }t j
� }t j d� j
j � �s�t||�� j� j�j� j� j� jd� �X}t||�� j
j|d� j� j� j� j� j � j � j � j � j� j ||� j!d�} || |� W dQ R X n
|dd� �j"S ) zCreate new archiveT)�fallbackc - sb t � }y"tjt� �}|j|j|jf� W n tk
r< Y nX � jj s�y$tj� jj
�}|j|j|jf� W n tk
r~ Y nX t j d� �x>� j
D �]2}|dk�r$� j}� j}� j}� j}��sy| j|||||�}W n6 tk
�r } zd}�jd|| � W Y d d } ~ X nX nd}�j||� q�tj
j|�}ytj|dd�}W n4 tk
�rv } z�jd|| � w�W Y d d } ~ X nX � j�r�|j}
nd }
�j| |�� j� j� j|||
� j �|d� |j|j|jf� q�W ��s^| j � j � j d� � j �r�| j!j"d d
� � j!� j#O _!� j!�r^� j#�r2t$t%�|d | id �� n,t&t't(| �t't)t(| j!�t(|�t't*j+d
�d� d S )NzProcessing files ...r �Ez%s: %sF)�follow_symlinks)� read_specialrS �st)�commentrD T)�finalr� )r� �extrazborg.output.stats)r� ),�setr �statrE �add�st_ino�st_dev�OSErrorr� �hostr� r� �debug�paths�
stdin_name�
stdin_mode�
stdin_user� stdin_groupZ
process_stdinr r� r� �normpath�one_file_system�_process�exclude_caches�exclude_if_present�keep_exclude_tagsrW �saverY rD r� �statsZ
show_progressr� rX rW rP �DASHESr� � STATS_HEADERr� r� ) r� r� � skip_inodesrX r� �mode�user�groupr� �e� restrict_dev)r~ rS r� r� r� r r� � create_inner sr
"
z(Archiver.do_create.<locals>.create_innerzCreating archive at "%s")r� r� r� Zpermit_adhoc_cacher� r� )r� r� �checkpoint_intervalr� �noatime�noctime� nobirthtimer� r� r� r� �chunker_params�startZstart_monotonicr� N)#ri r� �patternsr� r� �exclude_nodumprS r Zutcnowr& r' r� r� r� Z processedr r� r� r� �
no_cache_syncr� r� r r� rz r� r{ r| r} r� r� r� r~ r� r� )
r� r~ r� r� r� ry Zt0Z t0_monotonicr� r� r )r~ rS r� r� r� r� r( � s.
C
zArchiver.do_createc
) C s� �y�d}
|j |�r>| dkr�td�� tj|dd�} W dQ R X nT| jd|� |jsTdS | dkr~td�� tj|dd�} W dQ R X tj| j�}
|
s�dS | j| j f|kr�dS | dkp�| j | k}d}| j
r�td��& t || �tj @ r�| jd|� dS W dQ R X tj
| j��r | �s�|j|| |�}�nvtj| j��r |�r�t|||�}|�r�|
�s�|�r�| �s`|j|| � x.|D ]&}| j||||||||| |
| d� �qfW | jd|� dS | �s�|
�s�|j|| �}|�r�td�� tj|�}W dQ R X x<|D ]4}tjj|j�}| j||||||||| |
| d� �q�W �nttj| j��r�| �s�|
�sF|j|| �}nVytj|�}W n tk
�rn d}Y n X t|j�}|�r�|j|||�}n |j|| �}� n�tj| j��r�| �s�|
�s�|j|| �}n|j|| |�}n�tj | j��r| �s�|
�s|j || d �}n|j|| |�}n�tj | j��rL| �s�|
�s<|j || d
�}n|j|| |�}nFtj | j��r^dS tj | j��rpdS tj!| j��r�dS | j"d |� dS W n6 t#k
�r� } z| j"d ||� d
}W Y dd}~X nX |dk�r�| �s�d}nd}|
�s�| j||� dS )z�
Process *path* recursively according to the various parameters.
*st* (if given) is a *os.stat_result* object for *path*.
This should only raise on critical errors. Per-item errors must be handled within this method.
FNr] )rV �x�flags)rW rS �scandir�cr� zUnknown file type: %sz%s: %srU �?r )$�matchr r r] r� Z recurse_dir�S_ISDIR�st_moder_ r` r� rl � UF_NODUMP�S_ISREG� process_filerK Z process_dirrk r Zscandir_inorderr� ri �S_ISLNKZprocess_symlinkra r �S_ISFIFOZ process_fifo�S_ISCHRZ process_dev�S_ISBLK�S_ISSOCK�S_ISDOOR�S_ISPORTr� r )r� r� r� r� rl rm rn rs r� rx rW rS rX Zrecurse_excluded_dirZrecurser� Z tag_pathsZtag_pathZentriesZdirentri Z st_targetZspecialrw r r r� rk Y s�
zArchiver._processc s( �r� ��fdd�}n� �fdd�}|S )Nc s8 � j | j�o(tjj| jjtj��d � �}�| |� |S )N)r� r� r �sepr4 �split)r} �matched)r� � peek_and_store_hardlink_masters�strip_componentsr r� � item_filter� s *
z*Archiver.build_filter.<locals>.item_filterc s � j | j�}�| |� |S )N)r� r� )r} r� )r� r� r r� r� � s
r )r� r� r� r� r )r� r� r� r� � build_filter� s zArchiver.build_filterc s0 t j� dkr,tjd� t jjd!�r,tjd� | j|j|j�}|j }|j
}|j } |j }
|j
} |j} g }
|j� pp| ��s|t r�i nd � � �fd
d �}| j||| �}|r�td d
dd�}|jd� t� fdd�|j|�D ��}||_nd }�x�|j|�d� d�D �]h}|j}| �r*tjj|jtj�| d � �|_|j �s�xx|
�r�|jj|
d" j� �r�|
j d#�}y|j ||
d� W n8 t k
�r� } z| j dt |j�|� W Y d d }~X nX �q4W |�r�t!j"d�j#t |j�� y^| �r�|j |d|d� nBt$j%|j&��r |
j'|� |j ||
dd� n|j ||
| � | ||d � W n: t t(fk
�r` } z| j dt |�|� W Y d d }~X nX � q�W |�rv|j)� |j �s�tt*|
�d d d �}xj|
�r�|j+� |
j d$�}y|j ||
d� W n8 t k
�r� } z| j dt |j�|� W Y d d }~X nX �q�W x |j,� D ]}| j d |� �qW |�r*|j)� | j-S )%zExtract archive contents�asciiz_Warning: File system encoding is "ascii", extracting non-ascii filenames will not be supported.�linux�freebsd�netbsd�openbsd�darwinz^Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8Nc sJ t s�rF| rFt| j�rF| jdd�rFd| krF| jd�d f� | jd�< d S )N�hardlink_masterT�source�chunksr� )r rG rt r� )r} r� )�hardlink_masters�partial_extractr r� r� s z<Archiver.do_extract.<locals>.peek_and_store_hardlink_mastersz%5.1f%% Extracting: %sg�������?r% )r� �step�msgidz^Calculating total archive size for the progress indicator (might take long for large archives)c 3 s | ]}|j � �V qd S )N)�get_size)r| r} )r� r r� r� s z&Archiver.do_extract.<locals>.<genexpr>T)r� �preloadr� r )�stdoutz%s: %szborg.output.list)rS �piF)r� Z
restore_attrs)r� �sparser� Zstripped_components�
original_pathr� z%Setting directory permissions %3.0f%%zextract.permissions)�totalr� r� z#Include pattern '%s' never matched.)r� r� r� r� r� ���r� r� ).r� �getfilesystemencodingr� r� �platform�
startswithr� r� rd r� r� rS r� r� r� �emptyr r� rV �output�sum�
iter_itemsr� r� r r� r4 r� �pop� extract_itemr r� r> r� r� r� r] r� rt �appendr �finishr� �show� get_unmatched_include_patternsr� )r� r~ r� r� r� r� r� r� r� rS r� r� r� �dirsr� �filterr� �extracted_sizer} � orig_pathZdir_itemrw �patternr )r� r� r� r+ � s~
,
*
,zArchiver.do_extractc C s6 |j | _ d}|jdkrX|jjd�r(d}n"|jjd�r:d}n|jjd�rJd}tjd |� n|j}t|jd
�}|jd k}|r�|} |}
td d
�} t|t j
| d| d�} | s�t S | j }d }tj
|dtjd�}
| j|||
� |
j� |r�|j� |�r0tjd� | j� }|�rtjd|� t | _n tjd|� |
�r0| j� | jS )z$Export archive contents as a tarballN�autoz.tar.gzZgzipz.tar.bz2Zbzip2z.tar.xzZxzz'Automatically determined tar filter: %s�wbr T)�systemz--tar-filter: )�stdinr� Z
log_prefix�envzw|)Zfileobjrt r z/Done creating tar, waiting for filter to die...zA--tar-filter exited with code %d, output file is likely unusable!zfilter exited with code %d)r� �
tar_filter�tarfile�endswithr� rc r] r\ r[ �
subprocess�PIPEr& r� r� Z
GNU_FORMAT� _export_tarr� �waitr� r� )r� r~ r� r� r� r� r� Z tarstreamZtarstream_closeZ filteroutZfilterout_closer� Z
filterproc�tarr- r r r� �
do_export_tarF sL
zArchiver.do_export_tarc sl �j |j|j�}|j}|j}|j�|j� p.���r8i nd ���fdd�}�j||��}|r�tdddd���j d� t
�fdd �� j |�D ��} | �_ nd �� �fd
d ������fd d
�}
xz� j |�d�d�D ]d} | j
} �r�tjj| jtj��d � �| _
|
| | �\}
}|
r�|�r(tjd�jt| �� |j|
|� q�W ��rF�j� x |j� D ]}�jd|� �qPW �jS )Nc sD �r@| r@t | j�r@| jdd�r@d| kr@| jd�d f� | jd�< d S )Nr� Tr� r� r� )rG rt r� )r} r� )r� r� r r� r� � s z=Archiver._export_tar.<locals>.peek_and_store_hardlink_mastersz%5.1f%% Processing: %sg�������?r% )r� r� r� zCalculating sizec 3 s | ]}|j � �V qd S )N)r� )r| r} )r� r r� r� � s z'Archiver._export_tar.<locals>.<genexpr>c sL �j jdd� | jD �dd�}�r@t| j�g� t|� �fdd��S t|�S dS )zY
Return a file-like object that reads from the chunks of *item*.
c S s g | ]\}}}|�qS r r )r| Zchunk_id�_r r r� �
<listcomp>� s zEArchiver._export_tar.<locals>.item_content_stream.<locals>.<listcomp>T)Z is_preloadedc s �j t| �� d�S )N)Zincreaser� )r� r� )Z
read_bytes)r� r� r r� �<lambda>� s zCArchiver._export_tar.<locals>.item_content_stream.<locals>.<lambda>N)�pipeline�
fetch_manyr� r> r� rZ )r} Zchunk_iterator)r� r� )r� r� �item_content_stream� s z1Archiver._export_tar.<locals>.item_content_streamc s d}t j� }| j|_| jd |_tj| j�|_| j|_| j |_ | j
pFd|_ | j pRd|_
d|_tj| j�}|tjk�r.t j|_d| k�rtjj| jjtj��d� �}� dkr�|}n� j| jd|f�\}}|r�t j|_||_n<|dk �r,|| _| j� |_ �| �}d| jf� | jd��p|< n| j� |_ �| �}n�|tj k�rDt j |_n�|tj k�rbt j |_| j|_n�|tj!k�r�t j"|_tj#| j$�|_%tj&| j$�|_'nl|tj(k�r�t j)|_tj#| j$�|_%tj&| j$�|_'n:|tj*k�r�t j+|_n$�j,dt-| j�|� t.t/� d|fS ||fS )ad
Transform a Borg *item* into a tarfile.TarInfo object.
Return a tuple (tarinfo, stream), where stream may be a file-like object that represents
the file contents, if any, and is None otherwise. When *tarinfo* is None, the *item*
cannot be represented as a TarInfo object and should be skipped.
Ng e��A� r� z+%s: unsupported file type %o for tar export)0r� ZTarInfor� r� �mtimer] �S_IMODErt �uid�gidru �unamerv Zgname�linkname�S_IFMT�S_IFREGZREGTYPEr� r r� r4 r� r� r� ZLNKTYPEr� r� r: �S_IFDIRZDIRTYPE�S_IFLNKZSYMTYPE�S_IFBLKZBLKTYPE�majorZrdevZdevmajor�minorZdevminor�S_IFCHRZCHRTYPE�S_IFIFOZFIFOTYPEr� r> r* r% )r} r� �stream�tarinfoZmodebitsr� r� r� )r� r� r� r� r r� �item_to_tarinfo� s^
z-Archiver._export_tar.<locals>.item_to_tarinfoT)r� r� r� zborg.output.listz#Include pattern '%s' never matched.)r� r� rd r� r� r� r� r� rV r� r� r� r� r� r r� r4 r� r� r� r� r> Zaddfiler� r� r� r� )r� r~ r� r� r� r� r� r� r� r� r� r} r� r� r� r� r )r� r� r� r� r� r� r� r� r� � s>
L
zArchiver._export_tarc sT �fdd�� d&dd���fdd��
dd � � d
d � �d d
� � � fdd��� ��� �fdd��
��
�fdd���
fdd��� fdd��d'������ � ���f
dd� �dd � �d d � �����fd d �}|� t |||�j�jd!��� jjd"d��jjd"d#�k�p �j���s�jd$� �j�j�j �}|� �|� x |j
� D ]}�jd%|� �q8W �j S )(z Diff contents of two archivesc s$ |j j| �}|j j|�}� j||�S )N)r� r� r� )�
chunk_ids1�
chunk_ids2�archive1�archive2r� r� )r� r r� �fetch_and_compare_chunks! s z2Archiver.do_diff.<locals>.fetch_and_compare_chunksNc s> | j d�rd }n*� d k r2t� fdd�| jD ��}n| j� }|S )N�deletedc 3 s | ]}|j � kr|jV qd S )N)r r: )r| �chunk)� consider_idsr r� r� + s z;Archiver.do_diff.<locals>.sum_chunk_size.<locals>.<genexpr>)r� r� r� r� )r} r� r: r )r� r� �sum_chunk_size& s
z(Archiver.do_diff.<locals>.sum_chunk_sizec s" � j r| j| jfS | j| jfS d S )N)r� r� r� ru rv )r} )r~ r r� � get_owner0 s z#Archiver.do_diff.<locals>.get_ownerc S s d| krt j| j�S d gS d S )Nrt )r] �filemodert )r} r r r� �get_mode6 s z"Archiver.do_diff.<locals>.get_modec S s t | j�o| jd�|kS )Nr� )rG rt r� )r} r� r r r� �has_hardlink_master< s z-Archiver.do_diff.<locals>.has_hardlink_masterc S sX |j d�r*| j d� r*d| }d|i|fS | j d�rT|j d� rTd| }d|i|fS d S )Nr� zremoved r� zadded )r� )�item1�item2Z item_typeZchgr r r� �compare_presence? s z*Archiver.do_diff.<locals>.compare_presencec sD � | |d�}|d k r|S d| kr@d|kr@| j |j kr@ddidfS d S )N�linkr� r� z changed link)r� )r� r� Zpd)r� r r� � compare_linkG s
z&Archiver.do_diff.<locals>.compare_linkc sp | j d�|j d�krdS �r(| j|jkS �| ��|�kr<dS dd� | jD �}dd� |jD �}�||� �� S d S )Nr� Tc S s g | ]
}|j �qS r )r )r| r� r r r� r� \ s z>Archiver.do_diff.<locals>.contents_changed.<locals>.<listcomp>c S s g | ]
}|j �qS r )r )r| r� r r r� r� ] s )r� r� )r� r� r� r� )r� r� �can_compare_chunk_idsr� r� r r� �contents_changedQ s z*Archiver.do_diff.<locals>.contents_changedc
s� �||�r�|j d�r4�|�}d|d�djt|��fS |j d�r^�|�}d|d�djt|��fS � snddidfS d d
� |jD �}d d
� |jD �}|| }|| }�||�}�||�} d|| d �d
jt|ddd�t| ddd��fS d S )Nr� �added)r� r: z added {:>13}�removedzremoved {:>11}r� �modifiedc S s h | ]
}|j �qS r )r )r| r� r r r� � <setcomp>j s z<Archiver.do_diff.<locals>.compare_content.<locals>.<setcomp>c S s h | ]
}|j �qS r )r )r| r� r r r� r k s )r� r� r� z {:>9} {:>9}r T)Z precisionZsign)r� r r: r� )
r� r� r� Zszr� r� Z added_idsZ removed_idsr� r� )r� r� r� r r� �compare_content` s$
z)Archiver.do_diff.<locals>.compare_contentc sL � | �\}}� |�\}}||ks(||krHd||||d�dj ||||�fS d S )N�owner)r� Zold_userZ old_groupZnew_userZ new_groupz[{}:{} -> {}:{}])r )r� r� Zuser1Zgroup1Zuser2Zgroup2)r� r r� �
compare_ownert s
z'Archiver.do_diff.<locals>.compare_ownerc s8 | j |j kr4� | �}� |�}d||d�dj||�fS d S )Nrt )r� Zold_modeZnew_modez
[{} -> {}])rt r )r� r� Zmode1Zmode2)r� r r� � compare_mode{ s z&Archiver.do_diff.<locals>.compare_modeFc s� g }�||�r ||j d }�||�r4||j d }�|�d dksT�|�d dkrd|j�||�� d|kr�d|kr�|j�|||�� �|�d dks��|�d dkr�|j�||d�� �|�d dks؈|�d dkr�|j�||d�� �|�d d k�s�|�d d k�r |j�||d
�� �|�d d k�sD�|�d d k�rV|j�||d �� |�s||j�||�� |j�||�� d
d� |D �}|�r�t|�|f}� j�r�| j|� n� j�r|� n� |� dS )z�
Compare two items with identical paths.
:param deleted: Whether one of the items has been deleted
r r �lr� �dZ directoryr� Zblkdevr� Zchrdev�sZfifoc S s g | ] }|r|�qS r r )r| r� r r r� r� � s z;Archiver.do_diff.<locals>.compare_items.<locals>.<listcomp>N)r� r� r> �sort�
json_lines)r� r� r� r� r� r� �changesZ output_line)
r~ r r� r r r� r� r� �print_json_output�print_text_outputr r� �
compare_items� s8
$$
z'Archiver.do_diff.<locals>.compare_itemsc S s, | \}}t djdjdd� |D ��|�� d S )Nz {:<19} {}� c S s g | ] \}}|�qS r r )r| �j�txtr r r� r� � s z?Archiver.do_diff.<locals>.print_text_output.<locals>.<listcomp>)r� r r4 )�liner� �diffr r r� r
� s z+Archiver.do_diff.<locals>.print_text_outputc S s* | \}}t tj|dd� |D �d��� d S )Nc S s g | ] \}}|�qS r r )r| r r r r r� r� � s z?Archiver.do_diff.<locals>.print_json_output.<locals>.<listcomp>)r� r )r� r� r� )r r� r r r r� r � s z+Archiver.do_diff.<locals>.print_json_outputc s� �fdd��dd� ���fdd���� ����fdd�}t j� }t j� }g � i �g �x�t| j�fd d
��|j�fd d
���D ]�\}}|r�|r�|j|jkr�|||� q||r�|j|jd �}|r�|||� n
|||j< |r||j|jd �}|r�|||� q||||j< q|W td g d
d�} x<|j� D ]0}
|
j} | | _�| |
� ��| | |
�d d� �qW x<|j� D ]0} | j} | | _�| | � ��| | | �d d� �qVW x@� D ]8\}}�|��s�t��|��s�t���|j||�� �q�W �j �r؈ n�
}
xt
��D ]}|
|� �q�W d S )Nc s d| kp t | j� p | j� kS )Nr� )rG rt r� )r} )r� r r� �hardlink_master_seen� s zHArchiver.do_diff.<locals>.compare_archives.<locals>.hardlink_master_seenc S s | j dd�o d| ko t| j�S )Nr� Tr� )r� rG rt )r} r r r� �is_hardlink_master� s zFArchiver.do_diff.<locals>.compare_archives.<locals>.is_hardlink_masterc s" �| �s�|�r | |f� | j < d S )N)r� )r� r� )r� r r r� �update_hardlink_masters� s zKArchiver.do_diff.<locals>.compare_archives.<locals>.update_hardlink_mastersc sD �| |� �| � s �|� r.�j | |f� n� �| j| |�� d S )N)r� r� )r� r� )r �deferredr r� r� r r r� �compare_or_defer� s
zDArchiver.do_diff.<locals>.compare_archives.<locals>.compare_or_deferc s � j | j�S )N)r� r� )r} )r� r r� r� � s z<Archiver.do_diff.<locals>.compare_archives.<locals>.<lambda>c s � j | j�S )N)r� r� )r} )r� r r� r� � s Tr )r� r� rt )r� ) � collections� OrderedDictr r� r� r� rj �valuesr� r
�sorted)r� r� r� r Zorphans_archive1Zorphans_archive2r� r� Zmatching_orphanZ deleted_itemr� r� r� Z print_outputr )r~ r r r
)r r r� r r� r� r r� �compare_archives� s\
z*Archiver.do_diff.<locals>.compare_archives)r� r~ Tz�--chunker-params might be different between archives, diff will be slow.
If you know for certain that they are the same, pass --same-chunker-params to override this check.z#Include pattern '%s' never matched.)N)F) r r� r� �metadatar� �same_chunker_paramsr� r� r� rd r� r� ) r� r~ r� r� r� r� r r� r� r )r� r� r~ r� r r r� r r r� r� r� r� r� r� r r
r� r� r� �do_diff s6
0G
zArchiver.do_diff)r� r� r� c C s* |j |j� |j� |j� |j� | jS )zRename an existing archive)�renamer� r� r� r� )r� r~ r� r� r� r� r� r r r� � do_rename s
zArchiver.do_renamec C sf t |j|j|jdk |jf�}|jjp(|j}|rB|rB| jd� | j S |sJ|rV| j
||�S | j ||�S dS )z)Delete an existing repository or archivesNzFMixing archive filters and explicitly named archives is not supported.) r� r r r r r� r� �archivesr� r� �_delete_archives�_delete_repository)r� r~ r� Zarchive_filter_specifiedZexplicit_archives_specifiedr r r� r* s
zArchiver.do_deletec C s� |j }tj|tjjf�\}}|jjs*|jrVt|j�}|jjrL|j d|jj� t
|�}n$t
dd� |jj |�D ��}|sz| j S |j
dk�rVd}x�t|d�D ]x\} }
y|jj|
�} W n2 tk
r� t| _ tjdj|
| t|��� Y q�X d}|r�d nd
} tj| jt| �| t|��� q�W |�r$tjd � n,|�rF|j� |j� tjd � n
tjd
� | j S t� }
t||||j | j d���"}|�r�dnd}d}d}x�t|d�D ]�\} }
y|j|
}W n. tk
�r� tj|j|
| t|��� Y nVX tj|jt|�| t|��� |�s�t ||||
||j d�}|j |
|j |j
d� |d7 }�q�W |dk�r`|j� |j|j!d� |j� |j"�r�t#t$t%|
j&jd|
d�t'|�t$t(j)d�d� W dQ R X | j S )zDelete archivesr c s s | ] }|j V qd S )N)r� )r| r� r r r� r� 7 s z,Archiver._delete_archives.<locals>.<genexpr>rz Fr z Archive {} not found ({}/{}).TzWould delete: {} ({}/{})zDeleted archive: {} ({}/{})zFinished dry-run.z5Done. Run "borg check --repair" to clean up the mess.zAborted.)r� r� z Would delete archive: {} ({}/{})z Deleting archive: {} ({}/{}))r� r� )r� �forced)r z
Deleted data:)�labelrp zborg.output.stats)r� N)*rS rF r� � Operation�DELETEr� r� r# r� �insertr� �list_consideringr� r& � enumerater� �KeyErrorr% r� r� r r� r� r< r� r� r r r� r� r r� r# r rp rP rq rr �summaryr� r� r� )r� r~ r� rS r� r� r# �
archive_namesr� r< � archive_nameZcurrent_archiver� rp r� Z
msg_deleteZ
msg_not_foundZ delete_count� archive_infor� r r r� r$ + sp
zArchiver._delete_archivesc C s� |j }|j�s�|jdk�rXt|j�}|jj� }g }y0tj|tj �\}}t
|j �} |j dj
| �� W n" tk
r� d} |j d� Y nX |j t� |j dj
|�� |j dj
|�� |j d� |j d� | dk �r| dk�rx2|j jd gd
�D ]}
|j t|
�� q�W n
|j d � n
|j d � |j t� |j d
� dj|�}t|dddddd��sXt| _| jS |�s||j� tjd� tj|� n
tjd� |�s�tj|� tjd� n
tjd� | jS )zDelete a repositoryr z`You requested to completely DELETE the following repository *including* {} archives it contains:NzdYou requested to completely DELETE the following repository *including* all archives it may contain:zRepository ID: {}z Location: {}r� z Archives:r )r z/This repository seems to not have any archives.zXThis repository seems to have no manifest, so we can't tell anything about its contents.z8Type 'YES' if you understand this and want to continue: �
z Aborting.zInvalid answer, aborting.r� FZ"BORG_DELETE_I_KNOW_WHAT_I_AM_DOING)r� r� r� r� r� zRepository deleted.zWould delete repository.zCache deleted.zWould delete cache.)r� )rS �
cache_onlyr& r? r � _locationr� rF r� r� r� r# r� r r) rq r� r< r4 rN r& r� r� r� r� r r ) r� r~ r� rS r r� r� r� r� Z
n_archivesr1 r r r� r% q sT
zArchiver._delete_repositoryc C s� y ddl }W n4 tk
r@ } z| jdt|� � | jS d}~X nX tjj|j� sptj |jtj
tj B tj B � r�| jd|j � | jS | j
|�S )z:Mount archive or an entire repository as a FUSE filesystemr NzGborg mount not available: loading FUSE support failed [ImportError: %s]z+%s: Mountpoint must be a writable directory)Z borg.fuse� ImportErrorr� r� r� r r� �isdir�
mountpoint�access�R_OK�W_OK�X_OK� _do_mount)r� r~ �borgrw r r r� �do_mount� s .zArchiver.do_mountc C sx ddl m} t||d��T}||||||�}tjd� y|j|j|j|j� W n t k
rf t
| _ Y nX W d Q R X | j S )Nr )�FuseOperations)Zdecrypted_cachezMounting filesystem) Zfuser? rq r� r� �mountr7 �options�
foreground� RuntimeErrorr& r� )r� r~ r� r� r� r? Z cached_repoZ
operationsr r r� r< � s
zArchiver._do_mountc C s
t |j�S )z un-mount the FUSE filesystem)r^ r7 )r� r~ r r r� � do_umount� s zArchiver.do_umountc C sX |j jr.|jr | jd� | jS | j||||�S |jrD| jd� | jS | j||||�S dS )z#List archive or repository contentszKThe --json option is only valid for listing archives, not archive contents.zQThe --json-lines option is only valid for listing archive contents, not archives.N)r� r� r� r� r� �
_list_archiver
�_list_repository)r� r~ r� r� r� r r r� �do_list� s
zArchiver.do_listc s� | j � j� j��� jd k r"� j�n� jr.d�nd�� �����fdd�}tj��rzt���| jd��}||� W d Q R X n
|d d� | j S )Nz
{path}{NL}z<{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}c s\ t ���� jj| � jd�}t|�� jd�}x,|j�fdd��D ]}tjj |j
|�� q>W d S )N)r� r� )r
c s � j | j�S )N)r� r� )r} )r� r r� r� � s z=Archiver._list_archive.<locals>._list_inner.<locals>.<lambda>) r r� r� r� r7 r
r� r� r� r� � format_item)r� r� � formatterr} )r~ r r� r� r� r� r r� � _list_inner� s
z+Archiver._list_archive.<locals>._list_inner)r� )r� )
r� r� rd r �shortr7 Zformat_needs_cacher r� r� )r� r~ r� r� r� rJ r� r )r~ r r� r� r� r� r� rE � s
zArchiver._list_archivec C s� |j d k r|j }n|jr d}nd}t|||||jd�}g }x>|jj|�D ].}|jrd|j|j|�� qHtj j
|j |�� qHW |jr�t t
|d|id�� | jS )Nz
{archive}{NL}z {archive:<36} {time} [{id}]{NL})r� r# )r[ )r rK r8 r� r# r+ r� Z
get_item_datar� r� r� rH rX rW r� ) r� r~ r� r� r� r rI � output_datar1 r r r� rF � s
zArchiver._list_repository)r� r� c C sJ t |jj|j|j|jdk |jf�r4| j|||||�S | j|||||�S dS )z,Show archive details such as disk space usedN) r� r� r� r r r r �_info_archives�_info_repository)r� r~ r� r� r� r� r r r� �do_info s "zArchiver.do_infoc
C s dd� }|j jr |j jf}ntdd� |jj|�D ��}g }x�t|d�D ]�\} }
t||||
||jd�} | j� } |j r~|j
| � nHt t | d d��| d< || d �| d < t
tjd
�j� jf d |i| ��� | jr�P |j rFt|�| rFt
� qFW |j �r
tt||d |id
�� | jS )Nc S s t djdd� | D ���S )Nr c s s | ]}t j|�V qd S )N)r� r� )r| r� r r r� r� s zBArchiver._info_archives.<locals>.format_cmdline.<locals>.<genexpr>)r> r4 )�cmdliner r r� �format_cmdline s z/Archiver._info_archives.<locals>.format_cmdlinec s s | ] }|j V qd S )N)r� )r| r� r r r� r� s z*Archiver._info_archives.<locals>.<genexpr>r )r� r� Zduration)ZsecondsZ command_lineaF
Archive name: {name}
Archive fingerprint: {id}
Comment: {comment}
Hostname: {hostname}
Username: {username}
Time (start): {start}
Time (end): {end}
Duration: {duration}
Number of files: {stats[nfiles]}
Command line: {command_line}
Utilization of maximum supported archive size: {limits[max_archive_size]:.0%}
------------------------------------------------------------------------------
Original size Compressed size Deduplicated size
This archive: {stats[original_size]:>20s} {stats[compressed_size]:>20s} {stats[deduplicated_size]:>20s}
{cache}
r� r# )r� r[ )r� r� r� r# r+ r, r r� r� r� r� r9 r r� �textwrap�dedent�stripr r� r� rX rW )
r� r~ r� r� r� r� rQ r/ rL r< r0 r� r� r r r� rM s.
zArchiver._info_archivesc C s� t ||d|jjid�}|jr&t|� n�d}|jd
kr>|d7 }n|d|j 7 }|jjd�rh|d |j� 7 }||d
< tt j
d �j � j f t
|j�|jj� d �|��� tt� tt� tt|�� | jS )NZ security_dir)r� r[ z Encrypted: r� �
authenticatedZNozYes (%s)zkey filez
Key file: %s�
encryptionz�
Repository ID: {id}
Location: {location}
{encryption}
Cache: {cache.path}
Security dir: {security_dir}
)r r� )r� rU )rW Zsecurity_manager�dirr� rX r� r� r r� rR rS rT r r? r r4 r� rq rr r� r� )r� r~ r� r� r� r� r� rV r r r� rN C s(
zArchiver._info_repository)r� r� c s: t |j|j|j|j|j|j|j|jf�s8| j d� | j
S |j dk rN|j d |_ d}|j
j|j d| dgdd�}tjd | �j��fd
d �|D �� |r�� r�|d � d kr�� dd
� }ng }t� �� � fdd �|D �}g } |jr�| t||j�7 } |j�r
| t|d|j| �7 } |j�r&| t|d|j| �7 } |j�rB| t|d|j| �7 } |j�r^| t|d|j| �7 } |j�rz| t|d|j| �7 } |j�r�| t|d|j| �7 } |j�r�| t|d|j| �7 } t|�� B t| �t|�B }
t� } t|||| jd���F} tjd�}
|j�rt|
�}d }t t|
�ddd�}x�|D ]�}||
k�r�|j � |j �rZ|j�r�|
j dt |� � nF|j�r�|d
7 }|
j d t |�||f � t!||||j"| �j#| |j$d � n|j�r |
j d t |� � �q W |j%� |
�r�|j �r�|j&� |j'|j(d � | j'� |j)�r*t*t+t,| j-j.d | d!�t/| �t+tjd"�d#� W dQ R X | j
S )$z6Prune repository archives according to specified rulesz�At least one of the "keep-within", "keep-last", "keep-secondly", "keep-minutely", "keep-hourly", "keep-daily", "keep-weekly", "keep-monthly" or "keep-yearly" settings must be specified.Nr z\.checkpoint(\.\d+)?z(%s)?\Zr T)r Z match_endr �reversez(%s)\Zc s g | ]}� |j �r|�qS r )r� )r| �arch)�
is_checkpointr r� r� s s z%Archiver.do_prune.<locals>.<listcomp>r r c s g | ]}|� kr|�qS r r )r| rY )� checkpointsr r� r� | s z%Y-%m-%d %H:%M:%Sz%Y-%m-%d %H:%Mz %Y-%m-%d %Hz%Y-%m-%dz%G-%Vz%Y-%mz%Y)r� zborg.output.listzPruning archives %3.0f%%�prune)r� r� r� zWould prune: %szPruning archive: %s (%d/%d))r& zKeeping archive: %s)r z
Deleted data:)r' rp zborg.output.stats)r� )0r� �secondly�minutely�hourly�daily�weekly�monthly�yearly�withinr� r� r r r# r� �re�compile�searchr\ rB rC r r r� r� r� r� r� rV r� rS r� r< r r� r# r& r� r� r� r rp rP rq rr r. r r� )r� r~ r� r� r� Z
checkpoint_reZarchives_checkpointsZkeep_checkpointsr# ZkeepZ to_deleterp r� Z list_loggerZ
to_delete_lenZarchives_deletedr� r� r )r[ rZ r� �do_prunec s�
zArchiver.do_prune�tam� disable_tam� archives_tam)r� r� r� r� c C s� |j �r>tj|tjjf|jd�\}}t|||���}t� }x�|jj dgd�D ]�}|j
}t |�} |j |�}
|j
||
�} |j| dd�\} }
}|
�st| d�} dd� | jD �| _|j| j� dd �} |j| �}|j|| |� |j||� ||jf|j|j< td
| t|�f � qLtd | � qLW |j� |j� |j� W d Q R X �nR|j �r>tj|tjjf|jd�\}}|j �s~|j j d
d� �r�td� x&|jj dgd�D ]}tt |�� �q�W d|j d
< |j� |j� |j �rt |d��rd|_ |j!|j"� td� t |d��rtd|j#� � t |��s�t$|�}t%|d�j&� td� �nR|j'�r�tj|tj(dd�\}}t |��rtt)j*t$|�� |j �r�t |d��r�d|_ |j!|j"� td� t |d��r�td|j#� � d|j d
< |j� |j� n�t+|j,j-dd�}y|j.|j/|j0|j1d� W n0 t2k
�r2 } ztd| � W Y d d }~X nX t3|j,j-dd�}y|j.|j/|j0|j1d� W n0 t2k
�r� } ztd| � W Y d d }~X nX | j4S )z,upgrade a repository from a previous version)Zforce_tam_not_requiredr )r T)Z
internal_dictc S s g | ] }t |��qS r )rc )r| �argr r r� r� � s z'Archiver.do_upgrade.<locals>.<listcomp>s archive)�contextz Added archive TAM: %s -> [%s]zArchive TAM present: %sNs tam_requiredFzManifest contents:r
z Key updatedr z
Key location:r� zUpdated security database)r� )�inplacer� z warning: %s)5rk rF r� r( �CHECK�forcer r r# r� r r< r� �decryptZunpack_and_verify_archiverk rP Z pack_and_authenticate_metadataZas_dictZid_hashZ add_chunkZ chunk_decrefr r� r� r? r� r� ri Z tam_verified�configr r r
Z _passphraser r r� r� rj r� r �unlinkrx r� r� �upgraderS rn r� �NotImplementedErrorry r� )r� r~ r� r� r� r� rp r� Z
archive_idZarchive_formatted�cdatar= r� Zverifiedr� Znew_archive_idr1 r� r, rw r r r� �
do_upgrade� s�
zArchiver.do_upgrade)r� r� r� c C sR | j |j|j�}|j| _|j| _|jdk}|jdk}t||||||j|j|j |j
|j |||j |j
| j|j|j|jd�} |jjr�|jj}
| j|
�r�| jd|
� | jS | j|
|j|j�s�| jd� nl|jdk r�| jd� | jS xP|jjdgd �D ]<} | j }
| j|
��rq�t d
|
� | j|
|j�s�t j d |
� q�W |j�sL|j � |j!� |j!� | jS ) zRe-create archives�never�always)
rl rm rn r~ r� �
recompress�always_recompressr� rp Zfile_status_printerrz rS rD z;Refusing to work on temporary archive of prior recreate: %sz~Nothing to do. Archive was not processed.
Specify at least one pattern, PATH, --comment, re-compression or re-chunking option.Nz(--target: Need to specify single archiver )r Z
Processingz=Skipped archive %s: Nothing to do. Archive was not processed.)"r� r� rd r� r� rz r rl rm rn r~ r� r� rp r� rz rS rD r� r� Zis_temporary_archiver� r� �recreaterY r r# r� r� r� r� r� r� r� ) r� r~ r� r� r� r� r� rz r{ Z recreaterr� r� r r r� � do_recreate� sF
zArchiver.do_recreate)r� r� c
C s` |j tj�}|jtj|� t|d�r.|jj� tdd�}ztj |j
g|j |d�S |j � X dS )z:run a user specified command with the repository lock held�ioT)r� )r� N)
r� rF r �putr r~ Z
close_segmentr\ r� Zcall�commandr~ r� )r� r~ r� r= r� r r r� � do_with_lock) s
zArchiver.do_with_lockc s ddd�}ddd�}� fdd�}|j s~|jdkr@| jd � | jS y|jjd
�\}}W n( tk
r| |jrnd ppd }|j}Y nX |jr�tj� tj j
f�\}} t � || j � t
� | || j d
�}
�z.|jr�|
jj� |
jj} |
jj} |}
n� j} � fdd�} |}
|j�rB|
||dd� | j||� t| j|��dk�r:| j|� | � n�|j �rT|| � n�|j�r�|
|||j� || j� k�r�| j|� | j|||j� | � nNyt| j ||�� W n8 t j t j fk
�r� } zt|t j!d� t"S d}~X nX t#S |j�r�|
j$� X dS )z@get, set, and delete values in a repository or cache config fileNTc S s< | dkrt d��|dkrJ|rHy t|� W n t k
rF t d�d �Y nX n�|dkr�|r�y t|� W n t k
r� t d�d �Y nX |dkr�t|�td�k r�t d ��n |dkr�t|�tkr�t d
t ��nr|dkr�|r�|dkr�t d��nT|dk�r0|�r8y t|�}W n t d�d �Y nX t|�dk�r8t d��nt d��d S )Nr� zInvalid section�segments_per_dirz
Invalid value�max_segment_size�additional_free_spacer� r� z"Invalid value: storage_quota < 10Mz%Invalid value: max_segment_size >= %dr� �0r! r z$Invalid value, must be 64 hex digits� z Invalid name)r� )r� )r� r� r� )r� )r� r! )r )�
ValueError�intr; �MAX_SEGMENT_SIZE_LIMITr r� )�sectionr� �value� check_valueZbin_idr r r� �
repo_validateK s@
z)Archiver.do_config.<locals>.repo_validatec S s2 | dkrt d��|dkr&|r.t|� nt d��d S )Nr� zInvalid section�previous_locationz Invalid name)r� )r� )r� r/ )r� r� r� r� r r r� �cache_validaten s
z*Archiver.do_config.<locals>.cache_validatec s~ dt t�t t�d� j� jd�}td� xRdD ]J}| jd |d
d�}|d krf|j|�}|d krftd| ��td||f � q,W d S )Nr! r� )�versionr� r� r� r� r� z [repository]r� r� r� r� r� r� r r� F)rT zFThe repository config is missing the %s key which has no default valuez%s = %s)r� r� r� r� r� r� r )r� ZDEFAULT_SEGMENTS_PER_DIRr� r� r� r� r� r( )rr Zdefault_valuesr� r� )r� r r� � list_configw s
z'Archiver.do_config.<locals>.list_configz No config key name was provided.�.r� r� )r� c s � j � j� j�S )N)Z save_configr� rr r )r� r r� r� � s z$Archiver.do_config.<locals>.<lambda>F)r� r )r� )NT)NT)%r� r� r� r� r� r� r� rF r� r( �WRITEr r� r Z cache_configZ_configro rr r# Z
remove_optionr� rA Zremove_sectionr� ZsectionsZ add_sectionr\ r� r� � configparserZ
NoOptionErrorZNoSectionErrorr� r� r% r$ r� )r� r~ r� r� r� r� r� r� r� r� r� rr ro Zvalidaterw r )r� r� � do_configG s^
#
zArchiver.do_configc C s& t t� � t dtj� t dt� � tS )z6display system information for debugging / bug reportszCRC implementation:z Process ID:)r� rO r �__name__rm r$ )r� r~ r r r� �
do_debug_info� s
zArchiver.do_debug_infoc
C s� t ||||jj|jd�}xdt|jj�D ]T\}}|j||j|��}d|t |�f } t
d| � t | d��}
|
j |� W dQ R X q&W t
d� t
S )zAdump (decrypted, decompressed) archive items metadata (not: data))r� z
%06d_%s.items�Dumpingr� NzDone.)r r� r� r� r, r �itemsrq r� r? r� r� r� r$ ) r� r~ r� r� r� r� r< �item_idr= �filenamer> r r r� �do_debug_dump_archive_items� s
z$Archiver.do_debug_dump_archive_itemsc s� y|j j� t�jj� � W n" tk
r< tj�jj��Y nX d��fdd��� ����fdd�}t�j d��}||� W dQ R X t
S )z)dump decoded archive metadata (not: data)� c s t jtj| � d�d� d�S )N)�indentr )r )rR r� r� r� )r )r� r r� � do_indent� s z1Archiver.do_debug_dump_archive.<locals>.do_indentc s6 | j d� | j dtj�jj� d � | j d� | j �t� ��� | j d� �j� d �j� d ��}tj |t
dd�}| j d� | j �t|��� | j d� | j d � tj d
t
d �}d }xd|d
D ]X}�j|�j|��}|j |� x6|D ].}t|�}|r�d
}n
| j d� | j �|�� q�W q�W | j d� | j d� d S )Nz{
z
"_name": z,
z "_manifest_entry":
s id�surrogateescape)� object_hook�unicode_errorsz
"_meta":
z "_items": [
F)�use_listr� Ts itemsr2 z ]
}
)
r� r� r� r� r� r@ rq r� r_ �unpackbrH ZUnpackerZfeed)r> r= Zarchive_org_dictZunpackerr r� r} )�archive_meta_origr~ r� r� r� r r� r� � s0
z.Archiver.do_debug_dump_archive.<locals>.outputr� N) r# Z get_raw_dictr= r� r� r- r Z DoesNotExistr] r� r$ )r� r~ r� r� r� r� r> r )r� r~ r� r� r� r� r� �do_debug_dump_archive� s z Archiver.do_debug_dump_archivec C sT |j d|j|j��}ttj|tdd��}t|jd��}t j
||dd� W dQ R X t S )z dump decoded repository manifestNr� )r� r� r� r� )r� ) rq r� r r@ r` r� rH r] r� r� �dumpr$ )r� r~ r� r� r� r= �metar> r r r� �do_debug_dump_manifest s
z Archiver.do_debug_dump_manifestc s` ddl m} d
� fdd� }|jr�x.|j� D ]"\}}}}} |tkr*|||�� P q*W d}
xz|j� D ]n\}}}}} |tkr�||
||d|| d� n:|tkr�||
|dd || d� n |tkr�||
ddd
|| d� |
d7 }
q^W n�|jddd �} |j| d �}|||�� d} d}
xR|j t
| d �}
|
�sP |
d } x,|
D ]$}|j|�}||
||� |
d7 }
�q&W �qW t d � t S )zOdump (decrypted, decompressed) repo objects, repo index MUST be current/correctr )� key_factoryNc s� |d k r(|t jkr|nd }� j||�}nd}|d kr8dnd| }|d k rTdt|� nd} |d k rldt|� nd}
|d k r�dt|� nd} d| | |
|| f } td| � t| d��}
|
j|� W d Q R X d S )N� r� r� z%08d%s%s%s%s.objr� r� )rF r rq r� r? r� r� r� )r< r rv �tag�segment�offset�give_idr= Ztag_strZ segment_strZ
offset_strZid_strr� r> )r� r r� � decrypt_dump s
z6Archiver.do_debug_dump_repo_objs.<locals>.decrypt_dumpr r )r� r� r� �delr� )�limit�markerzDone.)NNNr� )
�
crypto.keyr� �ghostZscan_low_levelrt ru rv r� r� �scanrs r� r$ )r� r~ r� r� r� r rv r� r� r� r< �idsr� �resultr )r� r� �do_debug_dump_repo_objs s@
z Archiver.do_debug_dump_repo_objsc s2 d� � fdd�}|j }yF|jd�r4t|dd� �}n&|jd�rR|dd� jd�}ntd ��W n ttfk
rx d}Y nX |s�| jd
� tS d d lm } |j
d dd
�}|j |d �}|||�}d} d}
d} d} �xT|j t
| d
�}
|
s�P |
d } �x0|
D �]&}|j |�}|tjk�r|nd}|j||�}|
t|�d d� |dt|�d � }||k�r�|
t|�d � d� |dt|�d � � }|j|�}d| | j� |j� f }|||||� |j|�}|�r�|j|�}d| |j� |f }|||||� || } }
| d 7 } | d dkr�td| � q�W q�W td� tS )zMsearch for byte sequences in repo objects, repo index MUST be current/correctr� c sZ ||� |� }||t |� |t |� � � }td| |j� |j� |j� |||f � d S )Nz%s: %s %s %s == %r %r %r)r� r� �hex)r� �wantedr= r� Zbefore�after)rm r r� �
print_findingQ s z9Archiver.do_debug_search_repo_objs.<locals>.print_findingzhex:r� Nzstr:zutf-8zunsupported search termz6search term needs to be hex:123abc or str:foobar styler )r� )r� r� r r� z
%d %s | %sz %d %s #%di' z%d objects processed.zDone.r� )r� r� r �encoder� �UnicodeEncodeErrorr� r& r� r� r� r� r� rs rF r rq r� �findr� r9 r� r$ )r� r~ r� r� r� r� r� rv r� r� Z last_dataZlast_idr< r� r r� r= Z
boundary_datar� r� r9 r )rm r� �do_debug_search_repo_objsL s`
*
2
z"Archiver.do_debug_search_repo_objsc C s� |j }y t|�}W n tk
r2 td| � Y nbX y|j|�}W n" tjk
rd td| � Y n0X t|jd��}|j |� W dQ R X td| � t
S )z>get object contents from the repository and write it into filezobject id %s is invalid.zobject %s not found.r� Nzobject %s fetched.) r r r� r� r� rr �ObjectNotFoundr� r� r� r$ )r� r~ r� �hex_idr r= �fr r r� �do_debug_get_obj� s zArchiver.do_debug_get_objc C sd xV|j D ]L}t|d��}|j� }W dQ R X tj|�}|j|j� |� td|j� � qW |j � t
S )z(put file(s) contents into the repository�rbNzobject %s put.) rd r� �read�hashlibZsha256r Zdigestr� Z hexdigestr� r$ )r� r~ r� r� r� r= �hr r r� �do_debug_put_obj� s
zArchiver.do_debug_put_objc C s� d}x~|j D ]t}y t|�}W n tk
r< td| � Y q X y |j|� d}td| � W q tjk
r~ td| � Y q X q W |r�|j� td� tS )z3delete the objects with the given IDs from the repoFzobject id %s is invalid.Tzobject %s deleted.zobject %s not found.zDone.) r� r r� r� r# rr r� r� r$ )r� r~ r� r r� r r r r� �do_debug_delete_obj� s
z Archiver.do_debug_delete_obj)r� r� r� r� c C s� x�|j D ]v}y t|�}W n tk
r8 td| � Y qX y"|j| d }td||f � W q tk
r| td| � Y qX qW tS )z4display refcounts for the objects with the given IDszobject id %s is invalid.r z4object %s has %d referrers [info from chunks cache].z-object %s not found [info from chunks cache].)r� r r� r� r� r- r$ ) r� r~ r� r� r� r� r� r Zrefcountr r r� �do_debug_refcount_obj� s z Archiver.do_debug_refcount_objc C sd |j s|j|j� � z@t|j|j|jd�}t|jd��}t j
||dd� W dQ R X W d|j � X t S )zdump repository hints)�segments�compact�storage_quota_user� r� )r� N)
Z _active_txnZ prepare_txnZget_transaction_id�dictr� r� r� r] r� r� r� Zrollbackr$ )r� r~ r� Zhintsr> r r r� �do_debug_dump_hints� s
z Archiver.do_debug_dump_hintsc C sL ddl }|j�4 |j�" |jtj|jddd�|j� W dQ R X W dQ R X tS )z&convert Borg profile to Python profiler NFzutf-8)r� �encoding)�marshalr� �inputr� r_ �unpackr$ )r� r~ r� r r r� �do_debug_convert_profile� s 0z!Archiver.do_debug_convert_profile)r� r� c C s |j � tj |� | jS )zCBreak the repository lock (e.g. in case it was left by a dead borg.)Z
break_lockr r� )r� r~ r� r r r� �
do_break_lock� s
zArchiver.do_break_locka-
The path/filenames used as input for the pattern matching start from the
currently active recursion root. You usually give the recursion root(s)
when invoking borg and these can be either relative or absolute paths.
So, when you give `relative/` as root, the paths going into the matcher
will look like `relative/.../file.ext`. When you give `/absolute/` as
root, they will look like `/absolute/.../file.ext`. This is meant when
we talk about "full path" below.
File paths in Borg archives are always stored normalized and relative.
This means that e.g. ``borg create /path/to/repo ../some/path`` will
store all files as `some/path/.../file.ext` and ``borg create
/path/to/repo /home/user`` will store all files as
`home/user/.../file.ext`. Therefore, always use relative paths in your
patterns when matching archive content in commands like ``extract`` or
``mount``. Starting with Borg 1.2 this behaviour will be changed to
accept both absolute and relative paths.
A directory exclusion pattern can end either with or without a slash ('/').
If it ends with a slash, such as `some/path/`, the directory will be
included but not its content. If it does not end with a slash, such as
`some/path`, both the directory and content will be excluded.
File patterns support these styles: fnmatch, shell, regular expressions,
path prefixes and path full-matches. By default, fnmatch is used for
``--exclude`` patterns and shell-style is used for the ``--pattern`` option.
If followed by a colon (':') the first two characters of a pattern are
used as a style selector. Explicit style selection is necessary when a
non-default style is desired or when the desired pattern starts with
two alphanumeric characters followed by a colon (i.e. `aa:something/*`).
`Fnmatch <https://docs.python.org/3/library/fnmatch.html>`_, selector `fm:`
This is the default style for ``--exclude`` and ``--exclude-from``.
These patterns use a variant of shell pattern syntax, with '\*' matching
any number of characters, '?' matching any single character, '[...]'
matching any single character specified, including ranges, and '[!...]'
matching any character not specified. For the purpose of these patterns,
the path separator (backslash for Windows and '/' on other systems) is not
treated specially. Wrap meta-characters in brackets for a literal
match (i.e. `[?]` to match the literal character `?`). For a path
to match a pattern, the full path must match, or it must match
from the start of the full path to just before a path separator. Except
for the root path, paths will never end in the path separator when
matching is attempted. Thus, if a given pattern ends in a path
separator, a '\*' is appended before matching is attempted.
Shell-style patterns, selector `sh:`
This is the default style for ``--pattern`` and ``--patterns-from``.
Like fnmatch patterns these are similar to shell patterns. The difference
is that the pattern may include `**/` for matching zero or more directory
levels, `*` for matching zero or more arbitrary characters with the
exception of any path separator.
Regular expressions, selector `re:`
Regular expressions similar to those found in Perl are supported. Unlike
shell patterns regular expressions are not required to match the full
path and any substring match is sufficient. It is strongly recommended to
anchor patterns to the start ('^'), to the end ('$') or both. Path
separators (backslash for Windows and '/' on other systems) in paths are
always normalized to a forward slash ('/') before applying a pattern. The
regular expression syntax is described in the `Python documentation for
the re module <https://docs.python.org/3/library/re.html>`_.
Path prefix, selector `pp:`
This pattern style is useful to match whole sub-directories. The pattern
`pp:root/somedir` matches `root/somedir` and everything therein.
Path full-match, selector `pf:`
This pattern style is (only) useful to match full paths.
This is kind of a pseudo pattern as it can not have any variable or
unspecified parts - the full path must be given.
`pf:root/file.ext` matches `root/file.ext` only.
Implementation note: this is implemented via very time-efficient O(1)
hashtable lookups (this means you can have huge amounts of such patterns
without impacting performance much).
Due to that, this kind of pattern does not respect any context or order.
If you use such a pattern to include a file, it will always be included
(if the directory recursion encounters it).
Other include/exclude patterns that would normally match will be ignored.
Same logic applies for exclude.
.. note::
`re:`, `sh:` and `fm:` patterns are all implemented on top of the Python SRE
engine. It is very easy to formulate patterns for each of these types which
requires an inordinate amount of time to match paths. If untrusted users
are able to supply patterns, ensure they cannot supply `re:` patterns.
Further, ensure that `sh:` and `fm:` patterns only contain a handful of
wildcards at most.
Exclusions can be passed via the command line option ``--exclude``. When used
from within a shell, the patterns should be quoted to protect them from
expansion.
The ``--exclude-from`` option permits loading exclusion patterns from a text
file with one pattern per line. Lines empty or starting with the number sign
('#') after removing whitespace on both ends are ignored. The optional style
selector prefix is also supported for patterns loaded from a file. Due to
whitespace removal, paths with whitespace at the beginning or end can only be
excluded using regular expressions.
To test your exclusion patterns without performing an actual backup you can
run ``borg create --list --dry-run ...``.
Examples::
# Exclude '/home/user/file.o' but not '/home/user/file.odt':
$ borg create -e '*.o' backup /
# Exclude '/home/user/junk' and '/home/user/subdir/junk' but
# not '/home/user/importantjunk' or '/etc/junk':
$ borg create -e '/home/*/junk' backup /
# Exclude the contents of '/home/user/cache' but not the directory itself:
$ borg create -e /home/user/cache/ backup /
# The file '/home/user/cache/important' is *not* backed up:
$ borg create -e /home/user/cache/ backup / /home/user/cache/important
# The contents of directories in '/home' are not backed up when their name
# ends in '.tmp'
$ borg create --exclude 're:^/home/[^/]+\.tmp/' backup /
# Load exclusions from file
$ cat >exclude.txt <<EOF
# Comment line
/home/*/junk
*.tmp
fm:aa:something/*
re:^/home/[^/]+\.tmp/
sh:/home/*/.thumbnails
# Example with spaces, no need to escape as it is processed by borg
some file with spaces.txt
EOF
$ borg create --exclude-from exclude.txt backup /
A more general and easier to use way to define filename matching patterns exists
with the ``--pattern`` and ``--patterns-from`` options. Using these, you may
specify the backup roots (starting points) and patterns for inclusion/exclusion.
A root path starts with the prefix `R`, followed by a path (a plain path, not a
file pattern). An include rule starts with the prefix +, an exclude rule starts
with the prefix -, an exclude-norecurse rule starts with !, all followed by a pattern.
.. note::
Via ``--pattern`` or ``--patterns-from`` you can define BOTH inclusion and exclusion
of files using pattern prefixes ``+`` and ``-``. With ``--exclude`` and
``--exclude-from`` ONLY excludes are defined.
Inclusion patterns are useful to include paths that are contained in an excluded
path. The first matching pattern is used so if an include pattern matches before
an exclude pattern, the file is backed up. If an exclude-norecurse pattern matches
a directory, it won't recurse into it and won't discover any potential matches for
include rules below that directory.
.. note::
It's possible that a sub-directory/file is matched while parent directories are not.
In that case, parent directories are not backed up thus their user, group, permission,
etc. can not be restored.
Note that the default pattern style for ``--pattern`` and ``--patterns-from`` is
shell style (`sh:`), so those patterns behave similar to rsync include/exclude
patterns. The pattern style can be set via the `P` prefix.
Patterns (``--pattern``) and excludes (``--exclude``) from the command line are
considered first (in the order of appearance). Then patterns from ``--patterns-from``
are added. Exclusion patterns from ``--exclude-from`` files are appended last.
Examples::
# backup pics, but not the ones from 2018, except the good ones:
# note: using = is essential to avoid cmdline argument parsing issues.
borg create --pattern=+pics/2018/good --pattern=-pics/2018 repo::arch pics
# use a file with patterns:
borg create --patterns-from patterns.lst repo::arch
The patterns.lst file could look like that::
# "sh:" pattern style is the default, so the following line is not needed:
P sh
R /
# can be rebuild
- /home/*/.cache
# they're downloads for a reason
- /home/*/Downloads
# susan is a nice person
# include susans home
+ /home/susan
# also back up this exact file
+ pf:/home/bobby/specialfile.txt
# don't backup the other home directories
- /home/*
# don't even look in /proc
! /proc
You can specify recursion roots either on the command line or in a patternfile::
# these two commands do the same thing
borg create --exclude /home/bobby/junk repo::arch /home/bobby /home/susan
borg create --patterns-from patternfile.lst repo::arch
The patternfile::
# note that excludes use fm: by default and patternfiles use sh: by default.
# therefore, we need to specify fm: to have the same exact behavior.
P fm
R /home/bobby
R /home/susan
- /home/bobby/junk
This allows you to share the same patterns between multiple repositories
without needing to specify them on the command line.
r� a5
Repository (or Archive) URLs, ``--prefix``, ``--glob-archives``, ``--comment``
and ``--remote-path`` values support these placeholders:
{hostname}
The (short) hostname of the machine.
{fqdn}
The full name of the machine.
{reverse-fqdn}
The full name of the machine in reverse domain name notation.
{now}
The current local date and time, by default in ISO-8601 format.
You can also supply your own `format string <https://docs.python.org/3.7/library/datetime.html#strftime-and-strptime-behavior>`_, e.g. {now:%Y-%m-%d_%H:%M:%S}
{utcnow}
The current UTC date and time, by default in ISO-8601 format.
You can also supply your own `format string <https://docs.python.org/3.7/library/datetime.html#strftime-and-strptime-behavior>`_, e.g. {utcnow:%Y-%m-%d_%H:%M:%S}
{user}
The user name (or UID, if no name is available) of the user running borg.
{pid}
The current process ID.
{borgversion}
The version of borg, e.g.: 1.0.8rc1
{borgmajor}
The version of borg, only the major version, e.g.: 1
{borgminor}
The version of borg, only major and minor version, e.g.: 1.0
{borgpatch}
The version of borg, only major, minor and patch version, e.g.: 1.0.8
If literal curly braces need to be used, double them for escaping::
borg create /path/to/repo::{{literal_text}}
Examples::
borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
borg prune --prefix '{hostname}-' ...
.. note::
systemd uses a difficult, non-standard syntax for command lines in unit files (refer to
the `systemd.unit(5)` manual page).
When invoking borg from unit files, pay particular attention to escaping,
especially when using the now/utcnow placeholders, since systemd performs its own
%-based variable replacement even in quoted text. To avoid interference from systemd,
double all percent signs (``{hostname}-{now:%Y-%m-%d_%H:%M:%S}``
becomes ``{hostname}-{now:%%Y-%%m-%%d_%%H:%%M:%%S}``).
Z placeholdersa�
It is no problem to mix different compression methods in one repo,
deduplication is done on the source data chunks (not on the compressed
or encrypted data).
If some specific chunk was once compressed and stored into the repo, creating
another backup that also uses this chunk will not change the stored chunk.
So if you use different compression specs for the backups, whichever stores a
chunk first determines its compression. See also borg recreate.
Compression is lz4 by default. If you want something else, you have to specify what you want.
Valid compression specifiers are:
none
Do not compress.
lz4
Use lz4 compression. Very high speed, very low compression. (default)
zstd[,L]
Use zstd ("zstandard") compression, a modern wide-range algorithm.
If you do not explicitely give the compression level L (ranging from 1
to 22), it will use level 3.
Archives compressed with zstd are not compatible with borg < 1.1.4.
zlib[,L]
Use zlib ("gz") compression. Medium speed, medium compression.
If you do not explicitely give the compression level L (ranging from 0
to 9), it will use level 6.
Giving level 0 (means "no compression", but still has zlib protocol
overhead) is usually pointless, you better use "none" compression.
lzma[,L]
Use lzma ("xz") compression. Low speed, high compression.
If you do not explicitely give the compression level L (ranging from 0
to 9), it will use level 6.
Giving levels above 6 is pointless and counterproductive because it does
not compress better due to the buffer size used by borg - but it wastes
lots of CPU cycles and RAM.
auto,C[,L]
Use a built-in heuristic to decide per chunk whether to compress or not.
The heuristic tries with lz4 whether the data is compressible.
For incompressible data, it will not use compression (uses "none").
For compressible data, it uses the given C[,L] compression - with C[,L]
being any valid compression specifier.
Examples::
borg create --compression lz4 REPO::ARCHIVE data
borg create --compression zstd REPO::ARCHIVE data
borg create --compression zstd,10 REPO::ARCHIVE data
borg create --compression zlib REPO::ARCHIVE data
borg create --compression zlib,1 REPO::ARCHIVE data
borg create --compression auto,lzma,6 REPO::ARCHIVE data
borg create --compression auto,lzma ...
r� c C s� |j s|j� n�|j | jkr2tt| j|j �� n�|j |kr�|jrTt||j j� q�|jrvd ||j _||j j� q�||j j� nhg }|d|j g7 }|dg7 }|ddjt |j
� �� g7 }|ddjt | jj
� �� g7 }|j dj|�� | j S )NzNo help available on %s.zTry one of the following:z Commands: %sz, z Topics: %sr2 )
�topic�
print_help�helptextr� rd � epilog_only�epilog�
usage_onlyr4 r �keysr� r� )r� �parserZcommandsr~ Z msg_linesr r r� �do_helpA
s$
zArchiver.do_helpc C s |j � tS )z display infos about subcommand)r� r$ )r� r� r~ r r r� �do_subcommand_helpW
s zArchiver.do_subcommand_helpc C sr d d
ddg}x`t |d d � �D ]L\}}xB|D ]:\}}}|j|�r,|d k rX|j||�||< t|tjd � q,W q W |S )N�
--list-format�--format�EWarning: "--list-format" has been deprecated. Use "--format" instead.�--keep-tag-files�--keep-exclude-tags�SWarning: "--keep-tag-files" has been deprecated. Use "--keep-exclude-tags" instead.�--ignore-inode�jWarning: "--ignore-inode" has been deprecated. Use "--files-cache=ctime,size" or "...=mtime,size" instead.�--no-files-cache�VWarning: "--no-files-cache" has been deprecated. Use "--files-cache=disabled" instead.)r� )r� r� r� )r� r� r� )r� Nr� )r� Nr� )r, r� �replacer� r� r� )r� r~ Z deprecationsr< rl Zold_name�new_namer� r r r� �preprocess_args^
s
zArchiver.preprocess_argsc @ s2 e Zd ZdZdd� Zd dd�Zejd�dd �Zd
S ) zArchiver.CommonOptionsa�
Support class to allow specifying common options directly after the top-level command.
Normally options can only be specified on the parser defining them, which means
that generally speaking *all* options go after all sub-commands. This is annoying
for common options in scripts, e.g. --remote-path or logging options.
This class allows adding the same set of options to both the top-level parser
and the final sub-command parsers (but not intermediary sub-commands, at least for now).
It does so by giving every option's target name ("dest") a suffix indicating its level
-- no two options in the parser hierarchy can have the same target --
then, after parsing the command line, multiple definitions are resolved.
Defaults are handled by only setting them on the top-level parser and setting
a sentinel object in all sub-parsers, which then allows to discern which parser
supplied the option.
c C s( || _ || _t� | _t� | _t� | _dS )a�
*define_common_options* should be a callable taking one argument, which
will be a argparse.Parser.add_argument-like function.
*define_common_options* will be called multiple times, and should call
the passed function to define common options exactly the same way each time.
*suffix_precedence* should be a tuple of the suffixes that will be used.
It is ordered from lowest precedence to highest precedence:
An option specified on the parser belonging to index 0 is overridden if the
same option is specified on any parser with a higher index.
N)�define_common_options�suffix_precedencer� �common_optionsr\ �append_optionsr� �default_sentinel)r� r� r� r r r� r� �
s
z Archiver.CommonOptions.__init__Fc s8 ��j kst�� ���fdd�}|jd�� �j|� dS )a�
Add common options to *parser*.
*provide_defaults* must only be True exactly once in a parser hierarchy,
at the top level, and False on all lower levels. The default is chosen
accordingly.
*suffix* indicates the suffix to use internally. It also indicates
which precedence the *parser* has for common options. See *suffix_precedence*
of __init__.
c s� d|kr�|j dd� |d d ks$t�|d dk}|rZ�jj|d � |d g ksttd
��n�jj �t� �j|d � |d �7 < �s�|d | |d< |s��j|d < � j| |� d S ) N�dest�action�store�help� store_const�
store_true� store_falser� �defaultzCThe default is explicitly constructed as an empty list in resolve())r� r� r� r� r� r� )�
setdefaultr� r� r^ r� r\ r� � add_argument)r~ r� Z is_append)� common_group�provide_defaultsr� �suffixr r� r� �
s
z=Archiver.CommonOptions.add_common_group.<locals>.add_argumentzCommon optionsN)r� r� �add_argument_groupr� )r� r� r r� r� r )r� r� r� r r� �add_common_group�
s
z'Archiver.CommonOptions.add_common_group)r~ c
C s� xz| j D ]p}xj| jj|g �D ]X}|| }|}t||| j�}|| jk rPt|||� yt||� W q tk
rr Y q X q W qW x\| jD ]R}g }x<| j D ]2}|| }||kr�t||�} t||� |j | � q�W t|||� q�W dS )zd
Resolve the multiple definitions of each common option to the final value.
N)
r� r� r� r{ r� �setattr�delattr�AttributeErrorr� �extend)
r� r~ r r� Zmap_fromZmap_tor� Z option_valueZ extend_fromr r r r� �resolve�
s(
z Archiver.CommonOptions.resolveN)F) r� �
__module__� __qualname__�__doc__r� r r� Z Namespacer r r r r� �
CommonOptionsn
s
(r c 7
s� ddddddd���fdd �}d
d � }d d d
�dd�� � fdd�}ddd�dd�}t j| jdd d�}|jtj| j|�g g d� | j|�d7d �|_|j d d d d t
d!d"� |jj |ddd#� t jd | jd$�}|jg g d%� |jj |d� t jd | jd$�}|jg g d%� |jj |d� |d&�}|jd'k�rJ| j j
|_||_t j|_d(|_|} n.|jd)d*d+�}
|
jd,|gd | j j
|t jd(d-�} | j| j d.� | j d/d0t� d1d2� | j d3d4td5d2� | j d6d7d8d9d:d;� | j d<d=td>d?� || � | j d@dAdBtdCdD� || ddE� |jd'k�r
|S |dF�} |
jdG|gd | jj
| t jdHd-�} | j| jd.� | j dIdAdJdKdLdM� | j dNdAdOdKdPdM� | j dQdRd9dSd;� | j dTdUdVtd dWdX� |dY�} |
jdZ|gd | jj
| t jd[d-�} | j| jd.� | j d/d\d]d^td d_�d`da� | j dbdcdddedt� dfdg� | j dQdRd9dhd;� | j dTdUdVd tdidj� | j dkdld9dmd;� |dn�}
|
jdo|gd | jj
|
t jdpd-�} | j| jd.� | j d/d0d]d^t� dqda� | j drdsd9dtd;� | j dudvd9dwd;� | j dxdyd9dzd;� | j d{d|d9d}d;� | j d~dd9d�d;� || � |
jd�|gd d�d^t jd�d-�} | jd)d*d+�}| jtj| j | �d�� |d��}|jd�|gd | j j
|t jd�d-�} | j| j d.� | j d/d\d]d^td d_�d�� | j d�dAd]td�dD� | j d�d�d9d�d;� | j d�d�d9d�d;� |d��}|jd�|gd | j j
|t jd�d-�} | j| j d.� | j d/d\d]d^td d_�d�� | j d�dAd]td�dD� | j d�d�d9d�d;� |d��}|jd�|gd | j j
|t jd�d-�} | j| j d.� | j d/d\d]d^td d_�d�� |
jd�|gd | j j
|t jd�d-�} | j| j d.� | j d/d\d]d^td d_�d�� |d��}|jd�|gd | j!j
|t jd�d-�} | j| j!d.� | j d/d\d]d^td d_�d�� |d��}|
jd�|gd | j"j
|t jd�d-�} | j| j"d.� | j d�d�d�d9d�d;� | j d�d�d�d9d�d;� | j d�d�d9d�d;� | j d�d�d�d�d�� | j d�d9d�d�� | j d�d�d9d�d;� | j d�d�d�d�d;� | j d�d�d�d�d�d�� | j d�d�d�t#dd�d�� | j d�d�d�t$dd�d�� | j d�d�d�d�d̄ t%d�dX� || dd}|j d�d�d9d�d;� | j&d҃}|j d�d�d�d9d�d;� |j d�d�d9d�d;� |j d�d�d9d�d;� |j d�d�d9d�d;� |j d�d�d9d�d;� |j d�d�d9d�d;� |j d�d�d9d�d;� |j d�d�d9d�d;� |j d�d�d9d�d;� |j d�ddd�t't(d�t( dX� |j d�d�d9d�d;� | j&d��}|j d�d�d�t)d^d�d�� |j d�d�d�t*d d�dX� |j d��d �d�dt+�d�ddX� |j �d�d�dt,t-�dt- dX� |j �d �d
�d �d t.t.�d
��ddX� | j d/�dtdd_��dd2� | j d@dAdBt�ddD� |�d�}|
j�d|gd | j/j
|t j�dd-�} | j| j/d.� | j d�d�d9d�d;� | j d�d�d�d9�dd;� | j d�d�d9�dd;� | j d�d�d9�dd;� | j d�d�d9�dd;� | j d�d�d9�dd;� | j �d�dd9�d d;� | j �d �d d9�d d;� | j d/�dtdd_��d d2� | j d@dAdBtdCdD� || ddE� |�d!�}|
j�d"|gd | j0j
|t j�d#d-�} | j| j0d.� | j �d$�d%�d&�d'�d(� | j d�d�d9d�d;� | j d/�dtdd_��d)d2� | j �d*�d+�d,�d-� | j d@dAdBtdCdD� || ddE� |�d.�}|
j�d/|gd | j1j
|t j�d0d-�} | j| j1d.� | j d�d�d9�d1d;� | j �d2�d3d9�d4d;� | j �d5�d6d9�d7d;� | j �d8d9�d9d�� | j d/�d:tdd_��d;d2� | j �d<�d=t2� �d>d2� | j d@dAdBt�d?dD� || � |�d@�}|
j�dA|gd | j3j
|t j�dBd-�} | j| j3d.� | j d/�dtdd_��dCd2� | j �dD�dEt2� �dFd2� |�dG�}|
j�dH|gd | j4j
|t j�dId-�} | j| j4d.� | j d�d�d�d9�dJd;� | j d�d�d�d9�dKd;� | j �dL�dMd9�dNd;� | j �dO�dP�dQddR�dS� | j d~dd9d�d;� | j d/d0d]d^t� �dTda� | j �dU�ddB�dV�dW� || � |�dX�t5j6� t7j8�dY� t9j6� t7j8�dZ� t:j6� } |
j�d[|gd | j;j
| t j�d\d-�} | j| j;d.� | j �d]�d^d9�d_d;� | j �d`�da�db�dc�ddd�� | j d�d9�ded�� | j �d8d9�dfd�� | j d/d0d]d^t� �dgda� | j d@dAdBt�dhdD� || � || � |�di�} |
j�dj|gd | j<j
| t j�dkd-�} | j| j<d.� | j d3d4t�dld2� |�dm�} |
j�dn|gd | j=j
| t j�dod-�} | j| j=d.� | j d/d0d]d^t� �dpda� | j d�d9�dqd�� || � |�dr�} |
j�ds|gd | j>j
| t j�dtd-�} | j| j>d.� | j d/d\d]d^td d_��duda� |�dv�} |
j�dw|gd | j?j
| t j�dxd-�} | j| j?d.� | j d�d�d�d9�dJd;� | j �dO�dPd9�dyd;� | j d�d�d�d9�dKd;� | j d�d�d9�dzd;� | j �d{�d|�d}t@�d~�d� | j �d��d��d�t+dd��d�� | j �d��d�t+dd��d�� | j �d��d��d�t+dd��d�� | j �d��d��d�t+dd��d�� | j �d��d��d�t+dd��d�� | j �d��d��d�t+dd��d�� | j �d��d��d�t+dd��d�� || d d d� | j d~dd9d�d;� | j d/d\d]d^td d_��d�da� |�d��}!|
j�d�|gd | jAj
|!t j�d�d-�} | j| jAd.� | j d�d�d�d9�dJd;� | j �d��d�d9�d�d;� | j �dO�d�d9�d�d;� | j �d��d�d9�d�d;� | j �d��d�d9�d�d;� | j �d��d�d9�d�d;� | j d/d\d]d^td d_��d�da� |�d��}"|
j�d�|gd | jBj
|"t j| jBj
d-�} | j| jBd.� | j d�d�d9d�d;� | j d�d�d��d�d�� | j d�d�d�d9�d�d;� | j d�d�d�d9�d�d;� || dd | j&d��}|j �d��d��d�d t2� �d��d�� |j d��d �dt+�d�d�d�d�� |j d�d�d�t)d d�d�� |j d�d�d�t*d �d�dX� |j �d �d
�d �d t.t.�d
��ddX� |j �d�dd�d�d]�d��d��d8�d��d�� |j �d�d�dt,t-�d�t- dX� | j d/d0d]d^t� �d�da� | j d@dAdBt�d�dD� |�dŃ}#|
j�d�|gd | jCj
|#t j�d�d-�} | j| jCd.� | j d/d\td d_��d�d2� | j �dɐdʐdːd-� | j �d̐d�t jD�dΐdW� |�dσ}$|
j�d�|gd | jEj
|$t j�d�d-�} | j| jEd.� | j d��dҐd�d9�d�d;� | jF� }%|%j �d��dՐdHd9�d�d;� |%j �d�d��d[d9�d�d;� | j d/d\d]d^td �dِdڍ�d�da� | j �dDd�d]�dܐdW� | j �dݐd�d]�dߐdW� |
j�d�|gd �d�d�} | j �d�d�d9�d� | j �d�d�d9�d� | jtj| jG||
jH�d.� | j �d�d�td]�d�d� |�d�}&|
j�d�|gd �d�|&t j�d�d-�} | jd)d*d+�}'| jtj| j | �d�� |�d�}(|'j�dn|gd | jIj
|(t j�d�d-�} | j| jId.� |�d�})|'j�d�|gd | jJj
|)t j�d�d-�} | j| jJd.� | j d/�dtdd_��d�d2� |�d��}*|'j�d�|gd | jKj
|*t j�d�d-�} | j| jKd.� | j d/�dtdd_��d�d2� | j d�dAt�d�d2� |�d��}+|'j�d�|gd | jLj
|+t j�d�d-�} | j| jLd.� | j d/d\td d_��d�d2� | j d�dAt�d�d2� |�d��},|'j�d�|gd | jMj
|,t j�d�d-�} | j| jMd.� | j d/d\td d_��d�d2� | j �d �dd9�dd;� |�d�}-|'j�d|gd | jNj
|-t j�dd-�} | j| jNd.� | j d/d\td d_��dd2� | j �d�dt�d d2� |�d
�}.|'j�d |gd | jOj
|.t j�d d-�} | j| jOd.� | j d/d\td d_��d
d2� | j �d�dt�dd2� | j d�dAt�dd2� |�d�}/|'j�d|gd | jPj
|/t j�dd-�} | j| jPd.� | j d/d\td d_��d
d2� | j d@dA�dt�ddD� |�d�}0|'j�d|gd | jQj
|0t j�dd-�} | j| jQd.� | j d/d\td d_��d
d2� | j �d�d�dt�d dD� |�d �}1|'j�d |gd | jRj
|1t j�d d-�} | j| jRd.� | j d/d\td d_��d
d2� | j �d�d�dt�d dD� |�d!�}2|'j�d"|gd | jSj
|2t j�d#d-�} | j| jSd.� | j d/d\td d_��d�d2� | j d�dAt�d�d2� |�d$�}3|'j�d%|gd | jTj
|3t j�d&d-�} | j| jTd.� | j �d'�d(t jU�d)��d*d2� | j �d+�d,t jU�d-��d.d2� |�d/�}4|
j�d0|gd �d1|4t j�d1d-�} | jd)d*d+�}5| jtj| j | �d�� |�d2�}6|5j�d3|gd | jVj
|6t j�d4d-�} | j| jVd.� | j d/d\td d_��d5d2� | j d�dA�d6�d-� |S (9 Nu; "I am seeing ‘A’ (added) status for a unchanged file!?"z "Item flags"z"borg help patterns"z"borg help placeholders"z:Internals -> Data structures and file formats -> Key fileszborg key export --help)Za_status_oddityZlist_item_flagsZ
borg_patternsZborg_placeholdersZ key_filesZborg_key_exportc sh t j| �j� } y
tj}W n tk
r0 d}Y nX |dkrHdd� | D �} dj| �} |dkrdt| � �} | S )N� command-line� build_usagec S s g | ]}|j d �s|�qS )z.. man)r� )r| r r r r� r� s zAArchiver.build_parser.<locals>.process_epilog.<locals>.<listcomp>r2 )r r
)rR rS �
splitlinesr= Zdoc_moder r4 rd )r� rt )�rst_plain_text_referencesr r� �process_epilog�
s
z-Archiver.build_parser.<locals>.process_epilogc
S sf | ddddd� | dddd d
d d � | d
dddd
dd � | dddd
d
dd � | ddddddd
dd � | ddddd
dd � | ddd d g d d � | d d!d"d#d$d%� | d&d'd#d(d%� | d)d*d+t d,d-d.� | d/d0d1tjd2d3� | d4d5d#d6d%� | d7d8d#d9d%� | d:d;d<d=d>� td?d.� | d@dAdBdCdD� | dEdFdGt dHdI� | dJdKd#dLd%� | dMdNdOd dPdQ� | dRdSdTdUdD� d S )VNz-hz--helpr� z show this help message and exit)r� r� z
--critical� log_levelr� Zcriticalr� zwork on log level CRITICAL)r� r� �constr� r� z--errorr� zwork on log level ERRORz --warningz#work on log level WARNING (default)z--infoz-vz --verboser� zwork on log level INFOz--debugrc z,enable debug output, work on log level DEBUGz
--debug-topic�TOPIC� debug_topicsr� z�enable TOPIC debugging (can be specified multiple times). The logger path is borg.debug.<TOPIC> if TOPIC is not fully qualified.)�metavarr� r� r� r� z-pz
--progressr� r� zshow progress information)r� r� r� z
--log-jsonr� z>Output one JSON object per log line instead of formatted text.z --lock-wait�SECONDSr� r zRwait at most SECONDS for acquiring a repository/cache lock (default: %(default)d).)r r� r� r� r� z
--bypass-lockr� r� zBypass locking mechanism)r� r� r� r� z--show-version� show_versionzshow/log the borg versionz --show-rc�show_rcz show/log the return code (rc)z--umask�M�umaskc S s
t | d�S )N� )r� )r r r r� r� ' s zFArchiver.build_parser.<locals>.define_common_options.<locals>.<lambda>z9set umask to M (local and remote, default: %(default)04o)z
--remote-path�PATHZ remote_pathz;use PATH as borg executable on the remote (default: "borg"))r r� r� z--remote-ratelimitZRATEZremote_ratelimitzGset remote network upload rate limit in kiByte/s (default: 0=unlimited))r r� r� r� z--consider-part-filesr� z>treat part files like normal files (e.g. to list/extract them)z--debug-profile�FILE�
debug_profilez�Write execution profile in Borg format into FILE. For local use a Python-compatible file can be generated by suffixing FILE with ".pyprof".)r r� r� r� z--rshZRSHZrshzHUse this command to connect to the 'borg serve' process (default: 'ssh'))r� r� ZSUPPRESSZ
UMASK_DEFAULT)Zadd_common_optionr r r� r� sV
z4Archiver.build_parser.<locals>.define_common_optionsF)� tag_filesr� c S s� | ddddt ddd� | dd td
d � | d dtd
d � | ddtdd � |r�| ddddd� | ddddtdd� | dd d dd d� |r�| d d d!td"d#d$� d S )%Nz-ez --excludeZPATTERNr� r� z exclude paths matching PATTERN)r r� r� r� r� z--exclude-fromZ EXCLUDEFILEz4read exclude patterns from EXCLUDEFILE, one per line)r r� r� z --patternz&include/exclude paths matching PATTERNz--patterns-fromZ PATTERNFILEz<read include/exclude patterns from PATTERNFILE, one per linez--exclude-cachesrl r� z_exclude directories that contain a CACHEDIR.TAG file (http://www.bford.info/cachedir/spec.html))r� r� r� z--exclude-if-presentr� rm zYexclude directories that are tagged by containing a filesystem object with the given NAME)r r� r� r� r� z--keep-exclude-tagsz--keep-tag-filesrn zyif tag objects are specified with ``--exclude-if-present``, don't omit the tag objects themselves from the backup archivez--strip-components�NUMBERr� r ziRemove the specified number of leading path elements. Paths with fewer elements will be silently skipped.)r r� r� r� r� )rh rf re rg r� r� )Z
add_optionr r� r r r� �define_exclude_and_patterns5 s*
z:Archiver.build_parser.<locals>.define_exclude_and_patternsc s | j d�}� |jf|� |S )NzExclusion options)r r� )� subparserr� �
exclude_group)r! r r� �define_exclusion_groupR s
z5Archiver.build_parser.<locals>.define_exclusion_groupT)r �
first_lastc S s� | j dd�}|j� }|jddddtd dd� |jd d
d d td d
d� |rpd}|jdddt|djdjt�|�d� |r�|j� }|jddddt dd� |jddddt d d� d S ) NzArchive filtersz5Archive filters can be applied to repository targets.z-Pz--prefixZPREFIXr z6only consider archive names starting with this prefix.)r r� r� r� r� z-az--glob-archivesZGLOBr z�only consider archive names matching the glob. sh: rules apply, see "borg help patterns". ``--prefix`` and ``--glob-archives`` are mutually exclusive.rD z --sort-byZKEYSr zHComma-separated list of sorting keys; valid keys are: {}; default is: {}z, z--first�Nr r z:consider first N archives after other filters were applied)r r� r� r� r� z--lastr z9consider last N archives after other filters were applied)
r � add_mutually_exclusive_groupr� r0 r1 r3 r r4 r4 r+ )r" r r% Z
filters_grouprv Zsort_by_defaultr r r� � define_archive_filters_groupW s(
z;Archiver.build_parser.<locals>.define_archive_filters_groupzBorg - Deduplicated Backups)r� � description�add_help)r� rd r� � _maincommand� _midcommand� _subcommand)r� z-Vz --versionr� z %(prog)s z show version number and exit)r� r� r� )r� )r* r� )rd r� a�
This command mounts an archive as a FUSE filesystem. This can be useful for
browsing an archive or restoring individual files. Unless the ``--foreground``
option is given the command will run in the background until the filesystem
is ``umounted``.
The command ``borgfs`` provides a wrapper for ``borg mount``. This can also be
used in fstab entries:
``/path/to/repo /mnt/point fuse.borgfs defaults,noauto 0 0``
To allow a regular user to use fstab entries, add the ``user`` option:
``/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0``
For FUSE configuration and mount options, see the mount.fuse(8) manual page.
Borg's default behavior is to use the archived user and group names of each
file and map them to the system's respective user and group ids.
Alternatively, using ``numeric-ids`` will instead use the archived user and
group ids without any mapping.
The ``uid`` and ``gid`` mount options (implemented by Borg) can be used to
override the user and group ids of all files (i.e., ``borg mount -o
uid=1000,gid=1000``).
The man page references ``user_id`` and ``group_id`` mount options
(implemented by fuse) which specify the user and group id of the mount owner
(aka, the user who does the mounting). It is set automatically by libfuse (or
the filesystem if libfuse is not used). However, you should not specify these
manually. Unlike the ``uid`` and ``gid`` mount options which affect all files,
``user_id`` and ``group_id`` affect the user and group id of the mounted
(base) directory.
Additional mount options supported by borg:
- versions: when used with a repository mount, this gives a merged, versioned
view of the files in the archives. EXPERIMENTAL, layout may change in future.
- allow_damaged_files: by default damaged files (where missing chunks were
replaced with runs of zeros by borg check ``--repair``) are not readable and
return EIO (I/O error). Set this option to read such files.
- ignore_permissions: for security reasons the "default_permissions" mount
option is internally enforced by borg. "ignore_permissions" can be given to
not enforce "default_permissions".
The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users
to tweak the performance. It sets the number of cached data chunks; additional
memory usage can be up to ~8 MiB times this number. The default is the number
of CPU cores.
When the daemonized process receives a signal or crashes, it does not unmount.
Unmounting in these cases could cause an active rsync or similar process
to unintentionally delete data.
When running in the foreground ^C/SIGINT unmounts cleanly, but other
signals or crashes do not.
Zborgfszmount repositoryzrequired argumentsz <command>)�titler r@ )�parentsr* r) r� �formatter_classr� )r� r� ZREPOSITORY_OR_ARCHIVEz repository or archive to mount)r r� r� r7 Z
MOUNTPOINTzwhere to mount filesystemz-fz --foregroundrB r� z$stay in foreground, do not daemonize)r� r� r� z-orA zExtra mount options)r� r� r� rd r r z(paths to extract; patterns are supported)r �nargsr� r� )r� zm
This command starts a repository server process. This command is usually not used manually.
r� z start repository server processz--restrict-to-pathr� r� z�restrict repository access to PATH. Can be specified multiple times to allow the client access to several directories. Access to all sub-directories is granted implicitly; PATH doesn't need to directly point to a repository.)r r� r� r� z--restrict-to-repositoryr� a� restrict repository access. Only the repository located at PATH (no sub-directories are considered) is accessible. Can be specified multiple times to allow the client access to several repositories. Unlike ``--restrict-to-path`` sub-directories are not accessible; PATH needs to directly point at a repository location. PATH may be an empty directory or the last element of PATH may not exist, in which case the client may initialize a repository there.z
--append-onlyr� z0only allow appending to repository segment filesz--storage-quotaZQUOTAr� z�Override storage quota of the repository (e.g. 5G, 1.5T). When a new repository is initialized, sets the storage quota on the new repository as well. Default: no quota.)r r� r� r� r� a/
This command initializes an empty repository. A repository is a filesystem
directory containing the deduplicated data from zero or more archives.
Encryption mode TLDR
++++++++++++++++++++
The encryption mode can only be configured when creating a new repository -
you can neither configure it on a per-archive basis nor change the
encryption mode of an existing repository.
Use ``repokey``::
borg init --encryption repokey /path/to/repo
Or ``repokey-blake2`` depending on which is faster on your client machines (see below)::
borg init --encryption repokey-blake2 /path/to/repo
Borg will:
1. Ask you to come up with a passphrase.
2. Create a borg key (which contains 3 random secrets. See :ref:`key_files`).
3. Encrypt the key with your passphrase.
4. Store the encrypted borg key inside the repository directory (in the repo config).
This is why it is essential to use a secure passphrase.
5. Encrypt and sign your backups to prevent anyone from reading or forging them unless they
have the key and know the passphrase. Make sure to keep a backup of
your key **outside** the repository - do not lock yourself out by
"leaving your keys inside your car" (see :ref:`borg_key_export`).
For remote backups the encryption is done locally - the remote machine
never sees your passphrase, your unencrypted key or your unencrypted files.
Chunking and id generation are also based on your key to improve
your privacy.
6. Use the key when extracting files to decrypt them and to verify that the contents of
the backups have not been accidentally or maliciously altered.
Picking a passphrase
++++++++++++++++++++
Make sure you use a good passphrase. Not too short, not too simple. The real
encryption / decryption key is encrypted with / locked by your passphrase.
If an attacker gets your key, he can't unlock and use it without knowing the
passphrase.
Be careful with special or non-ascii characters in your passphrase:
- Borg processes the passphrase as unicode (and encodes it as utf-8),
so it does not have problems dealing with even the strangest characters.
- BUT: that does not necessarily apply to your OS / VM / keyboard configuration.
So better use a long passphrase made from simple ascii chars than one that
includes non-ascii stuff or characters that are hard/impossible to enter on
a different keyboard layout.
You can change your passphrase for existing repos at any time, it won't affect
the encryption/decryption key or other secrets.
More encryption modes
+++++++++++++++++++++
Only use ``--encryption none`` if you are OK with anyone who has access to
your repository being able to read your backups and tamper with their
contents without you noticing.
If you want "passphrase and having-the-key" security, use ``--encryption keyfile``.
The key will be stored in your home directory (in ``~/.config/borg/keys``).
If you do **not** want to encrypt the contents of your backups, but still
want to detect malicious tampering use ``--encryption authenticated``.
To normally work with ``authenticated`` repos, you will need the passphrase, but
there is an emergency workaround, see ``BORG_WORKAROUNDS=authenticated_no_key`` docs.
If ``BLAKE2b`` is faster than ``SHA-256`` on your hardware, use ``--encryption authenticated-blake2``,
``--encryption repokey-blake2`` or ``--encryption keyfile-blake2``. Note: for remote backups
the hashing is done on your local machine.
.. nanorst: inline-fill
+----------+---------------+------------------------+--------------------------+
| Hash/MAC | Not encrypted | Not encrypted, | Encrypted (AEAD w/ AES) |
| | no auth | but authenticated | and authenticated |
+----------+---------------+------------------------+--------------------------+
| SHA-256 | none | `authenticated` | repokey |
| | | | keyfile |
+----------+---------------+------------------------+--------------------------+
| BLAKE2b | n/a | `authenticated-blake2` | `repokey-blake2` |
| | | | `keyfile-blake2` |
+----------+---------------+------------------------+--------------------------+
.. nanorst: inline-replace
Modes `marked like this` in the above table are new in Borg 1.1 and are not
backwards-compatible with Borg 1.0.x.
On modern Intel/AMD CPUs (except very cheap ones), AES is usually
hardware-accelerated.
BLAKE2b is faster than SHA256 on Intel/AMD 64-bit CPUs
(except AMD Ryzen and future CPUs with SHA extensions),
which makes `authenticated-blake2` faster than `none` and `authenticated`.
On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster
than BLAKE2b-256 there. NEON accelerates AES as well.
Hardware acceleration is always used automatically when available.
`repokey` and `keyfile` use AES-CTR-256 for encryption and HMAC-SHA256 for
authentication in an encrypt-then-MAC (EtM) construction. The chunk ID hash
is HMAC-SHA256 as well (with a separate key).
These modes are compatible with Borg 1.0.x.
`repokey-blake2` and `keyfile-blake2` are also authenticated encryption modes,
but use BLAKE2b-256 instead of HMAC-SHA256 for authentication. The chunk ID
hash is a keyed BLAKE2b-256 hash.
These modes are new and *not* compatible with Borg 1.0.x.
`authenticated` mode uses no encryption, but authenticates repository contents
through the same HMAC-SHA256 hash as the `repokey` and `keyfile` modes (it uses it
as the chunk ID hash). The key is stored like `repokey`.
This mode is new and *not* compatible with Borg 1.0.x.
`authenticated-blake2` is like `authenticated`, but uses the keyed BLAKE2b-256 hash
from the other blake2 modes.
This mode is new and *not* compatible with Borg 1.0.x.
`none` mode uses no encryption and no authentication. It uses SHA256 as chunk
ID hash. This mode is not recommended, you should rather consider using an authenticated
or authenticated/encrypted mode. This mode has possible denial-of-service issues
when running ``borg create`` on contents controlled by an attacker.
Use it only for new repositories where no encryption is wanted **and** when compatibility
with 1.0.x is important. If compatibility with 1.0.x is not important, use
`authenticated-blake2` or `authenticated` instead.
This mode is compatible with Borg 1.0.x.
Zinitzinitialize empty repositoryZ
REPOSITORYr� r� )r� zrepository to create)r r1 r� r� r� z-ez --encryptionZMODErV z)select encryption key mode **(required)**)r r� Zrequired�choicesr� z%create an append-only mode repositoryzKSet storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.)r r� r� r� r� z--make-parent-dirsr� zOcreate the parent directories of the repository directory, if they are missing.a%
The check command verifies the consistency of a repository and the corresponding archives.
check --repair is a potentially dangerous function and might lead to data loss
(for kinds of corruption it is not capable of dealing with). BE VERY CAREFUL!
Pursuant to the previous warning it is also highly recommended to test the
reliability of the hardware running this software with stress testing software
such as memory testers. Unreliable hardware can also lead to data loss especially
when this command is run in repair mode.
First, the underlying repository data files are checked:
- For all segments, the segment magic header is checked.
- For all objects stored in the segments, all metadata (e.g. CRC and size) and
all data is read. The read data is checked by size and CRC. Bit rot and other
types of accidental damage can be detected this way.
- In repair mode, if an integrity error is detected in a segment, try to recover
as many objects from the segment as possible.
- In repair mode, make sure that the index is consistent with the data stored in
the segments.
- If checking a remote repo via ``ssh:``, the repo check is executed on the server
without causing significant network traffic.
- The repository check can be skipped using the ``--archives-only`` option.
Second, the consistency and correctness of the archive metadata is verified:
- Is the repo manifest present? If not, it is rebuilt from archive metadata
chunks (this requires reading and decrypting of all metadata and data).
- Check if archive metadata chunk is present; if not, remove archive from manifest.
- For all files (items) in the archive, for all chunks referenced by these
files, check if chunk is present. In repair mode, if a chunk is not present,
replace it with a same-size replacement chunk of zeroes. If a previously lost
chunk reappears (e.g. via a later backup), in repair mode the all-zero replacement
chunk will be replaced by the correct chunk. This requires reading of archive and
file metadata, but not data.
- In repair mode, when all the archives were checked, orphaned chunks are deleted
from the repo. One cause of orphaned chunks are input file related errors (like
read errors) in the archive creation process.
- In verify-data mode, a complete cryptographic verification of the archive data
integrity is performed. This conflicts with ``--repository-only`` as this mode
only makes sense if the archive checks are enabled. The full details of this mode
are documented below.
- If checking a remote repo via ``ssh:``, the archive check is executed on the
client machine because it requires decryption, and this is always done client-side
as key access is needed.
- The archive checks can be time consuming; they can be skipped using the
``--repository-only`` option.
The ``--verify-data`` option will perform a full integrity verification (as opposed to
checking the CRC32 of the segment) of data, which means reading the data from the
repository, decrypting and decompressing it. This is a cryptographic verification,
which will detect (accidental) corruption. For encrypted repositories it is
tamper-resistant as well, unless the attacker has access to the keys. It is also very
slow.
r zverify repositoryz-repository or archive to check consistency ofz--repository-onlyr z only perform repository checksz--archives-onlyr
z only perform archives checksz
--verify-datar z`perform cryptographic archive data integrity verification (conflicts with ``--repository-only``)z--repairr� z+attempt to repair any inconsistencies foundz --save-spacer z!work slower, but using less spacer� z+Manage a keyfile or repokey of a repositoryzmanage repository key)r� a�
If repository encryption is used, the repository is inaccessible
without the key. This command allows to backup this essential key.
Note that the backup produced does not include the passphrase itself
(i.e. the exported key stays encrypted). In order to regain access to a
repository, one needs both the exported key and the original passphrase.
There are two backup formats. The normal backup format is suitable for
digital storage as a file. The ``--paper`` backup format is optimized
for printing and typing in while importing, with per line checks to
reduce problems with manual input.
For repositories using keyfile encryption the key is saved locally
on the system that is capable of doing backups. To guard against loss
of this key, the key needs to be backed up independently of the main
data backup.
For repositories using the repokey encryption the key is saved in the
repository in the config file. A backup is thus not strictly needed,
but guards against the repository becoming inaccessible if the file
is damaged for some reason.
Examples::
borg key export /path/to/repo > encrypted-key-backup
borg key export --paper /path/to/repo > encrypted-key-backup.txt
borg key export --qr-html /path/to/repo > encrypted-key-backup.html
# Or pass the output file as an argument instead of redirecting stdout:
borg key export /path/to/repo encrypted-key-backup
borg key export --paper /path/to/repo encrypted-key-backup.txt
borg key export --qr-html /path/to/repo encrypted-key-backup.html
r z export repository key for backup)r r1 r� r� r� zwhere to store the backupz--paperr z8Create an export suitable for printing and later type-inz --qr-htmlr zFCreate an html file suitable for printing and later type-in or qr scanaN
This command allows to restore a key previously backed up with the
export command.
If the ``--paper`` option is given, the import will be an interactive
process in which each line is checked for plausibility before
proceeding to the next line. For this format PATH must not be given.
�importz!import repository key from backupz+path to the backup ('-' to read from stdin)z8interactively import from a backup done with ``--paper``a�
The key files used for repository encryption are optionally passphrase
protected. This command can be used to change this passphrase.
Please note that this command only changes the passphrase, but not any
secret protected by it (like e.g. encryption/MAC keys or chunker seed).
Thus, changing the passphrase after passphrase and borg key got compromised
does not protect future (nor past) backups to the same repository.
zchange-passphrasez change repository passphrasea�
This command migrates a repository from passphrase mode (removed in Borg 1.0)
to repokey mode.
You will be first asked for the repository passphrase (to open it in passphrase
mode). This is the same passphrase as you used to use for this repo before 1.0.
It will then derive the different secrets from this passphrase.
Then you will be asked for a new passphrase (twice, for safety). This
passphrase will be used to protect the repokey (which contains these same
secrets in encrypted form). You may use the same passphrase as you used to
use, but you may also use a different one.
After migrating to repokey mode, you can change the passphrase at any time.
But please note: the secrets will always stay the same and they could always
be derived from your (old) passphrase-mode passphrase.
zmigrate-to-repokeyz-migrate passphrase-mode repository to repokeya�"
This command creates a backup archive containing all files found while recursively
traversing all paths specified. Paths are added to the archive as they are given,
that means if relative paths are desired, the command has to be run from the correct
directory.
When giving '-' as path, borg will read data from standard input and create a
file 'stdin' in the created archive from that data. See section *Reading from
stdin* below for details.
The archive will consume almost no disk space for files or parts of files that
have already been stored in other archives.
The archive name needs to be unique. It must not end in '.checkpoint' or
'.checkpoint.N' (with N being a number), because these names are used for
checkpoints and treated in special ways.
In the archive name, you may use the following placeholders:
{now}, {utcnow}, {fqdn}, {hostname}, {user} and some others.
Backup speed is increased by not reprocessing files that are already part of
existing archives and weren't modified. The detection of unmodified files is
done by comparing multiple file metadata values with previous values kept in
the files cache.
This comparison can operate in different modes as given by ``--files-cache``:
- ctime,size,inode (default)
- mtime,size,inode (default behaviour of borg versions older than 1.1.0rc4)
- ctime,size (ignore the inode number)
- mtime,size (ignore the inode number)
- rechunk,ctime (all files are considered modified - rechunk, cache ctime)
- rechunk,mtime (all files are considered modified - rechunk, cache mtime)
- disabled (disable the files cache, all files considered modified - rechunk)
inode number: better safety, but often unstable on network filesystems
Normally, detecting file modifications will take inode information into
consideration to improve the reliability of file change detection.
This is problematic for files located on sshfs and similar network file
systems which do not provide stable inode numbers, such files will always
be considered modified. You can use modes without `inode` in this case to
improve performance, but reliability of change detection might be reduced.
ctime vs. mtime: safety vs. speed
- ctime is a rather safe way to detect changes to a file (metadata and contents)
as it can not be set from userspace. But, a metadata-only change will already
update the ctime, so there might be some unnecessary chunking/hashing even
without content changes. Some filesystems do not support ctime (change time).
E.g. doing a chown or chmod to a file will change its ctime.
- mtime usually works and only updates if file contents were changed. But mtime
can be arbitrarily set from userspace, e.g. to set mtime back to the same value
it had before a content change happened. This can be used maliciously as well as
well-meant, but in both cases mtime based cache modes can be problematic.
The mount points of filesystems or filesystem snapshots should be the same for every
creation of a new archive to ensure fast operation. This is because the file cache that
is used to determine changed files quickly uses absolute filenames.
If this is not possible, consider creating a bind mount to a stable location.
The ``--progress`` option shows (from left to right) Original, Compressed and Deduplicated
(O, C and D, respectively), then the Number of files (N) processed so far, followed by
the currently processed path.
When using ``--stats``, you will get some statistics about how much data was
added - the "This Archive" deduplicated size there is most interesting as that is
how much your repository will grow. Please note that the "All archives" stats refer to
the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually
exclusive because the data is not actually compressed and deduplicated during a dry run.
For more help on include/exclude patterns, see the :ref:`borg_patterns` command output.
For more help on placeholders, see the :ref:`borg_placeholders` command output.
.. man NOTES
The ``--exclude`` patterns are not like tar. In tar ``--exclude`` .bundler/gems will
exclude foo/.bundler/gems. In borg it will not, you need to use ``--exclude``
'\*/.bundler/gems' to get the same effect.
In addition to using ``--exclude`` patterns, it is possible to use
``--exclude-if-present`` to specify the name of a filesystem object (e.g. a file
or folder name) which, when contained within another folder, will prevent the
containing folder from being backed up. By default, the containing folder and
all of its contents will be omitted from the backup. If, however, you wish to
only include the objects specified by ``--exclude-if-present`` in your backup,
and not include any other contents of the containing folder, this can be enabled
through using the ``--keep-exclude-tags`` option.
The ``-x`` or ``--one-file-system`` option excludes directories, that are mountpoints (and everything in them).
It detects mountpoints by comparing the device number from the output of ``stat()`` of the directory and its
parent directory. Specifically, it excludes directories for which ``stat()`` reports a device number different
from the device number of their parent. Be aware that in Linux (and possibly elsewhere) there are directories
with device number different from their parent, which the kernel does not consider a mountpoint and also the
other way around. Examples are bind mounts (possibly same device number, but always a mountpoint) and ALL
subvolumes of a btrfs (different device number from parent but not necessarily a mountpoint). Therefore when
using ``--one-file-system``, one should make doubly sure that the backup works as intended especially when using
btrfs. This is even more important, if the btrfs layout was created by someone else, e.g. a distribution
installer.
.. _list_item_flags:
Item flags
++++++++++
``--list`` outputs a list of all files, directories and other
file system items it considered (no matter whether they had content changes
or not). For each item, it prefixes a single-letter flag that indicates type
and/or status of the item.
If you are interested only in a subset of that output, you can give e.g.
``--filter=AME`` and it will only show regular files with A, M or E status (see
below).
A uppercase character represents the status of a regular file relative to the
"files" cache (not relative to the repo -- this is an issue if the files cache
is not used). Metadata is stored in any case and for 'A' and 'M' also new data
chunks are stored. For 'U' all data chunks refer to already existing chunks.
- 'A' = regular file, added (see also :ref:`a_status_oddity` in the FAQ)
- 'M' = regular file, modified
- 'U' = regular file, unchanged
- 'E' = regular file, an error happened while accessing/reading *this* file
A lowercase character means a file type other than a regular file,
borg usually just stores their metadata:
- 'd' = directory
- 'b' = block device
- 'c' = char device
- 'h' = regular file, hardlink (to already seen inodes)
- 's' = symlink
- 'f' = fifo
Other flags used include:
- 'i' = backup data was read from standard input (stdin)
- '-' = dry run, item was *not* backed up
- 'x' = excluded, item was *not* backed up
- '?' = missing status code (if you see this, please file a bug report!)
Reading from stdin
++++++++++++++++++
To read from stdin, specify ``-`` as path and pipe directly to borg::
backup-vm --id myvm --stdout | borg create REPO::ARCHIVE -
Note that piping to borg creates an archive even if the command piping
to borg exits with a failure. In this case, **one can end up with
truncated output being backed up**.
Reading from stdin yields just a stream of data without file metadata
associated with it, and the files cache is not needed at all. So it is
safe to disable it via ``--no-files-cache`` and speed up backup
creation a bit.
By default, the content read from stdin is stored in a file called 'stdin'.
Use ``--stdin-name`` to change the name.
r� z
create backupz-nz --dry-runrS z do not create a backup archivez-sz--statsrp z(print statistics for the created archivez--listr� z/output verbose list of items (files, dirs, ...)z--filterZ STATUSCHARSr� zEonly display items with the given status characters (see description))r r� r� z--jsonz*output stats as JSON. Implies ``--stats``.)r� r� z--no-cache-syncr� zNexperimental: do not synchronize the cache. Implies not using the files cache.z--no-files-cacher� r� zIdo not load/update the file metadata cache used to detect unchanged filesz --stdin-namer� re r� z5use NAME in archive for stdin data (default: "stdin"))r r� r� r� z --stdin-userZUSERrg r z>set user USER in archive for stdin data (default: %(default)r)z
--stdin-groupZGROUPrh z@set group GROUP in archive for stdin data (default: %(default)r)z --stdin-moder rf c S s
t | d�S )Nr )r� )r r r r� r� $ s z'Archiver.build_parser.<locals>.<lambda>z@set mode to M in archive for stdin data (default: %(default)04o))r z--exclude-nodumpr� z exclude files flagged NODUMPzFilesystem optionsz-xz--one-file-systemrj z�stay in the same file system and do not store mount points of other file systems. This might behave different from your expectations, see the docs.z--numeric-ownerr� z-only store numeric user and group identifiersz --noatimer{ z do not store atime into archivez --noctimer| z do not store ctime into archivez
--nobirthtimer} z3do not store birthtime (creation date) into archivez --nobsdflagsr� zDdo not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archivez--noaclsr� z'do not read and store ACLs into archivez
--noxattrsr� z)do not read and store xattrs into archivez--ignore-inoder� zLignore inode data in the file metadata cache used to detect unchanged files.z
--files-cacher� z(operate files cache in MODE. default: %sz--read-specialrW z�open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.zArchive optionsz --commentrY �COMMENTz!add a comment text to the archive)r� r r� r� r� z --timestampZ TIMESTAMPrD z�manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). Alternatively, give a reference file/directory.z-cz--checkpoint-intervalr rz i z6write checkpoint every SECONDS seconds (Default: 1800)z--chunker-paramsZPARAMSr~ zuspecify the chunker parameters (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: %d,%d,%d,%dz-Cz
--compressionZ COMPRESSIONr� Zlz4z`select compression algorithm, see the output of the "borg help compression" command for details.ZARCHIVEz?name of archive to create (must be also a valid directory name)zpaths to archivea�
This command extracts the contents of an archive. By default the entire
archive is extracted but a subset of files and directories can be selected
by passing a list of ``PATHs`` as arguments. The file selection can further
be restricted by using the ``--exclude`` option.
For more help on include/exclude patterns, see the :ref:`borg_patterns` command output.
By using ``--dry-run``, you can do all extraction steps except actually writing the
output data: reading metadata and data chunks from the repo, checking the hash/hmac,
decrypting, decompressing.
``--progress`` can be slower than no progress display, since it makes one additional
pass over the archive metadata.
.. note::
Currently, extract always writes into the current working directory ("."),
so make sure you ``cd`` to the right place before calling ``borg extract``.
When parent directories are not extracted (because of using file/directory selection
or any other reason), borg can not restore parent directories' metadata, e.g. owner,
group, permission, etc.
r% zextract archive contentsz do not actually change any filesz,only obey numeric user and group identifiersz4do not extract/set bsdflags (e.g. NODUMP, IMMUTABLE)zdo not extract/set ACLszdo not extract/set xattrsz--stdoutr� z"write all extracted data to stdoutz--sparser� z7create holes in output sparse file from all-zero chunkszarchive to extracta�
This command creates a tarball from an archive.
When giving '-' as the output FILE, Borg will write a tar stream to standard output.
By default (``--tar-filter=auto``) Borg will detect whether the FILE should be compressed
based on its file extension and pipe the tarball through an appropriate filter
before writing it to FILE:
- .tar.gz: gzip
- .tar.bz2: bzip2
- .tar.xz: xz
Alternatively a ``--tar-filter`` program may be explicitly specified. It should
read the uncompressed tar stream from stdin and write a compressed/filtered
tar stream to stdout.
The generated tarball uses the GNU tar format.
export-tar is a lossy conversion:
BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported.
Timestamp resolution is limited to whole seconds, not the nanosecond resolution
otherwise supported by Borg.
A ``--sparse`` option (as found in borg extract) is not supported.
By default the entire archive is extracted but a subset of files and directories
can be selected by passing a list of ``PATHs`` as arguments.
The file selection can further be restricted by using the ``--exclude`` option.
For more help on include/exclude patterns, see the :ref:`borg_patterns` command output.
``--progress`` can be slower than no progress display, since it makes one additional
pass over the archive metadata.
z
export-tarzcreate tarball from archivez --tar-filterr� r� z#filter program to pipe data through)r� r� r� zarchive to exportr� r z0output tar file. "-" to write to stdout instead.)r r� a�
This command finds differences (file contents, user/group/mode) between archives.
A repository location and an archive name must be specified for REPO::ARCHIVE1.
ARCHIVE2 is just another archive name in same repository (no repository location
allowed).
For archives created with Borg 1.1 or newer diff automatically detects whether
the archives are created with the same chunker params. If so, only chunk IDs
are compared, which is very fast.
For archives prior to Borg 1.1 chunk contents are compared by default.
If you did not create the archives with different chunker params,
pass ``--same-chunker-params``.
Note that the chunker params changed from Borg 0.xx to 1.0.
For more help on include/exclude patterns, see the :ref:`borg_patterns` command output.
r z$find differences in archive contentsz0only consider numeric user and group identifiersz--same-chunker-paramsr z%Override check of chunker parameters.z--sortr z#Sort the output lines by file path.z --json-linesz Format output as JSON Lines. zREPO::ARCHIVE1z%repository location and ARCHIVE1 namer� ZARCHIVE2z.ARCHIVE2 name (no repository location allowed)zEpaths of items inside the archives to compare; patterns are supportedzu
This command renames an archive in the repository.
This results in a different archive ID.
r! zrename archivezarchive to renamer� ZNEWNAMEzthe new archive name to usea#
This command deletes an archive from the repository or the complete repository.
Disk space is reclaimed accordingly. If you delete the complete repository, the
local cache for it (if any) is also deleted.
When using ``--stats``, you will get some statistics about how much data was
deleted - the "Deleted data" deduplicated size there is most interesting as
that is how much your repository will shrink.
Please note that the "All archives" stats refer to the state after deletion.
r# zdelete archivezdo not change repositoryz(print statistics for the deleted archivez --cache-onlyr3 z4delete only the local cache for the given repositoryz--forcer& r9 z`force deletion of corrupted archives, use ``--force --force`` in case ``--force`` does not work.)r� r� r� r� z repository or archive to deleter# zarchives to delete)r r1 r� ah
This command lists the contents of a repository or an archive.
For more help on include/exclude patterns, see the :ref:`borg_patterns` command output.
.. man NOTES
The FORMAT specifier syntax
+++++++++++++++++++++++++++
The ``--format`` option uses python's `format string syntax
<https://docs.python.org/3.8/library/string.html#formatstrings>`_.
Examples:
::
$ borg list --format '{archive}{NL}' /path/to/repo
ArchiveFoo
ArchiveBar
...
# {VAR:NUMBER} - pad to NUMBER columns.
# Strings are left-aligned, numbers are right-aligned.
# Note: time columns except ``isomtime``, ``isoctime`` and ``isoatime`` cannot be padded.
$ borg list --format '{archive:36} {time} [{id}]{NL}' /path/to/repo
ArchiveFoo Thu, 2021-12-09 10:22:28 [0b8e9a312bef3f2f6e2d0fc110c196827786c15eba0188738e81697a7fa3b274]
$ borg list --format '{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}' /path/to/repo::ArchiveFoo
-rw-rw-r-- user user 1024 Thu, 2021-12-09 10:22:17 file-foo
...
# {VAR:<NUMBER} - pad to NUMBER columns left-aligned.
# {VAR:>NUMBER} - pad to NUMBER columns right-aligned.
$ borg list --format '{mode} {user:>6} {group:>6} {size:<8} {mtime} {path}{extra}{NL}' /path/to/repo::ArchiveFoo
-rw-rw-r-- user user 1024 Thu, 2021-12-09 10:22:17 file-foo
...
The following keys are always available:
zN
Keys available only when listing archives in a repository:
zI
Keys available only when listing files in an archive:
r� z#list archive or repository contentsz--shortrK z-only print file/directory names, nothing elsez--formatz
--list-formatZFORMATr z�specify format for file or archive listing (default for files: "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}"; for archives: "{archive:<36} {time} [{id}]{NL}")a Only valid for listing repository contents. Format output as JSON. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "barchive" key is therefore not available.a Only valid for listing archive contents. Format output as JSON Lines. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available.z)repository or archive to list contents ofz%paths to list; patterns are supportedz�
This command un-mounts a FUSE filesystem that was mounted with ``borg mount``.
This is a convenience wrapper that just calls the platform-specific shell
command - usually this is either umount or fusermount -u.
r^ zumount repositoryz&mountpoint of the filesystem to umounta{
This command displays detailed information about the specified archive or repository.
Please note that the deduplicated sizes of the individual archives do not add
up to the deduplicated size of the repository ("all archives"), because the two
are meaning different things:
This archive / deduplicated size = amount of data stored ONLY for this archive
= unique chunks of this archive.
All archives / deduplicated size = amount of data stored in the repo
= all chunks in the repository.
Borg archives can only contain a limited amount of file metadata.
The size of an archive relative to this limit depends on a number of factors,
mainly the number of files, the lengths of paths and other metadata stored for files.
This is shown as *utilization of maximum supported archive size*.
r� z&show repository or archive informationz2repository or archive to display information aboutzformat output as JSONz�
This command breaks the repository and cache locks.
Please use carefully and only while no borg process (on any machine) is
trying to access the Cache or the Repository.
z
break-lockz break repository and cache locksz'repository for which to break the locksa�
The prune command prunes a repository by deleting all archives not matching
any of the specified retention options. This command is normally used by
automated backup scripts wanting to keep a certain number of historic backups.
This retention policy is commonly referred to as
`GFS <https://en.wikipedia.org/wiki/Backup_rotation_scheme#Grandfather-father-son>`_
(Grandfather-father-son) backup rotation scheme.
Also, prune automatically removes checkpoint archives (incomplete archives left
behind by interrupted backup runs) except if the checkpoint is the latest
archive (and thus still needed). Checkpoint archives are not considered when
comparing archive counts against the retention limits (``--keep-X``).
If a prefix is set with -P, then only archives that start with the prefix are
considered for deletion and only those archives count towards the totals
specified by the rules.
Otherwise, *all* archives in the repository are candidates for deletion!
There is no automatic distinction between archives representing different
contents. These need to be distinguished by specifying matching prefixes.
If you have multiple sequences of archives with different data sets (e.g.
from different machines) in one shared repository, use one prune call per
data set that matches only the respective archives using the -P option.
The ``--keep-within`` option takes an argument of the form "<int><char>",
where char is "H", "d", "w", "m", "y". For example, ``--keep-within 2d`` means
to keep all archives that were created within the past 48 hours.
"1m" is taken to mean "31d". The archives kept with this option do not
count towards the totals specified by any other options.
A good procedure is to thin out more and more the older your backups get.
As an example, ``--keep-daily 7`` means to keep the latest backup on each day,
up to 7 most recent days with backups (days without backups do not count).
The rules are applied from secondly to yearly, and backups selected by previous
rules do not count towards those of later rules. The time that each backup
starts is used for pruning purposes. Dates and times are interpreted in
the local timezone, and weeks go from Monday to Sunday. Specifying a
negative number of archives to keep means that there is no limit.
The ``--keep-last N`` option is doing the same as ``--keep-secondly N`` (and it will
keep the last N archives under the assumption that you do not create more than one
backup archive in the same second).
When using ``--stats``, you will get some statistics about how much data was
deleted - the "Deleted data" deduplicated size there is most interesting as
that is how much your repository will shrink.
Please note that the "All archives" stats refer to the state after pruning.
r\ zprune archivesz_force pruning of corrupted archives, use ``--force --force`` in case ``--force`` does not work.z/output verbose list of archives it keeps/prunesz
--keep-withinZINTERVALrd z+keep all archives within this time interval)r r� r� r� z --keep-lastz--keep-secondlyr] z#number of secondly archives to keep)r� r� r� r� z--keep-minutelyr^ z#number of minutely archives to keepz-Hz
--keep-hourlyr_ z!number of hourly archives to keepz-dz --keep-dailyr` z number of daily archives to keepz-wz
--keep-weeklyra z!number of weekly archives to keepz-mz--keep-monthlyrb z"number of monthly archives to keepz-yz
--keep-yearlyrc z!number of yearly archives to keepzrepository to pruneal
Upgrade an existing, local Borg repository.
When you do not need borg upgrade
+++++++++++++++++++++++++++++++++
Not every change requires that you run ``borg upgrade``.
You do **not** need to run it when:
- moving your repository to a different place
- upgrading to another point release (like 1.0.x to 1.0.y),
except when noted otherwise in the changelog
- upgrading from 1.0.x to 1.1.x,
except when noted otherwise in the changelog
Borg 1.x.y upgrades
+++++++++++++++++++
Archive TAM authentication:
Use ``borg upgrade --archives-tam REPO`` to add archive TAMs to all
archives that are not TAM authenticated yet.
This is a convenient method to just trust all archives present - if
an archive does not have TAM authentication yet, a TAM will be added.
Archives created by old borg versions < 1.0.9 do not have TAMs.
Archives created by newer borg version should have TAMs already.
If you have a high risk environment, you should not just run this,
but first verify that the archives are authentic and not malicious
(== have good content, have a good timestamp).
Borg 1.2.5+ needs all archives to be TAM authenticated for safety reasons.
This upgrade needs to be done once per repository.
Manifest TAM authentication:
Use ``borg upgrade --tam REPO`` to require manifest authentication
introduced with Borg 1.0.9 to address security issues. This means
that modifying the repository after doing this with a version prior
to 1.0.9 will raise a validation error, so only perform this upgrade
after updating all clients using the repository to 1.0.9 or newer.
This upgrade should be done on each client for safety reasons.
If a repository is accidentally modified with a pre-1.0.9 client after
this upgrade, use ``borg upgrade --tam --force REPO`` to remedy it.
If you routinely do this you might not want to enable this upgrade
(which will leave you exposed to the security issue). You can
reverse the upgrade by issuing ``borg upgrade --disable-tam REPO``.
See
https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability
for details.
Attic and Borg 0.xx to Borg 1.x
+++++++++++++++++++++++++++++++
This currently supports converting an Attic repository to Borg and also
helps with converting Borg 0.xx to 1.0.
Currently, only LOCAL repositories can be upgraded (issue #465).
Please note that ``borg create`` (since 1.0.0) uses bigger chunks by
default than old borg or attic did, so the new chunks won't deduplicate
with the old chunks in the upgraded repository.
See ``--chunker-params`` option of ``borg create`` and ``borg recreate``.
``borg upgrade`` will change the magic strings in the repository's
segments to match the new Borg magic strings. The keyfiles found in
$ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and
copied to $BORG_KEYS_DIR or ~/.config/borg/keys.
The cache files are converted, from $ATTIC_CACHE_DIR or
~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the
cache layout between Borg and Attic changed, so it is possible
the first backup after the conversion takes longer than expected
due to the cache resync.
Upgrade should be able to resume if interrupted, although it
will still iterate over all segments. If you want to start
from scratch, use `borg delete` over the copied repository to
make sure the cache files are also removed::
borg delete borg
Unless ``--inplace`` is specified, the upgrade process first creates a backup
copy of the repository, in REPOSITORY.before-upgrade-DATETIME, using hardlinks.
This requires that the repository and its parent directory reside on same
filesystem so the hardlink copy can work.
This takes longer than in place upgrades, but is much safer and gives
progress information (as opposed to ``cp -al``). Once you are satisfied
with the conversion, you can safely destroy the backup copy.
WARNING: Running the upgrade in place will make the current
copy unusable with older version, with no way of going back
to previous versions. This can PERMANENTLY DAMAGE YOUR
REPOSITORY! Attic CAN NOT READ BORG REPOSITORIES, as the
magic strings have changed. You have been warned.rt zupgrade repository formatz --inplacern z^rewrite repository in place, with no chance of going back to older versions of the repository.rp z
Force upgradez--tamri zIEnable manifest authentication (in key and cache) (Borg 1.0.9 and later).z
--disable-tamrj z3Disable manifest authentication (in key and cache).z--archives-tamrk z(add TAM authentication for all archives.z%path to the repository to be upgradeda5
Recreate the contents of existing archives.
recreate is a potentially dangerous function and might lead to data loss
(if used wrongly). BE VERY CAREFUL!
``--exclude``, ``--exclude-from``, ``--exclude-if-present``, ``--keep-exclude-tags``
and PATH have the exact same semantics as in "borg create", but they only check
for files in the archives and not in the local file system. If PATHs are specified,
the resulting archives will only contain files from these PATHs.
Note that all paths in an archive are relative, therefore absolute patterns/paths
will *not* match (``--exclude``, ``--exclude-from``, PATHs).
``--recompress`` allows to change the compression of existing data in archives.
Due to how Borg stores compressed size information this might display
incorrect information for archives that were not recreated at the same time.
There is no risk of data loss by this.
``--chunker-params`` will re-chunk all files in the archive, this can be
used to have upgraded Borg 0.xx or Attic archives deduplicate with
Borg 1.x archives.
**USE WITH CAUTION.**
Depending on the PATHs and patterns given, recreate can be used to permanently
delete files from archives.
When in doubt, use ``--dry-run --verbose --list`` to see how patterns/PATHS are
interpreted. See :ref:`list_item_flags` in ``borg create`` for details.
The archive being recreated is only removed after the operation completes. The
archive that is built during the operation exists at the same time at
"<ARCHIVE>.recreate". The new archive will have a different archive ID.
With ``--target`` the original archive is not replaced, instead a new archive is created.
When rechunking space usage can be substantial, expect at least the entire
deduplicated size of the archives using the previous chunker params.
When recompressing expect approx. (throughput / checkpoint-interval) in space usage,
assuming all chunks are recompressed.
If you recently ran borg check --repair and it had to fix lost chunks with all-zero
replacement chunks, please first run another backup for the same data and re-run
borg check --repair afterwards to heal any archives that had lost chunks which are
still generated from the input data.
Important: running borg recreate to re-chunk will remove the chunks_healthy
metadata of all items with replacement chunks, so healing will not be possible
any more after re-chunking (it is also unlikely it would ever work: due to the
change of chunking parameters, the missing chunk likely will never be seen again
even if you still have the data that produced it).
r| zRonly display items with the given status characters (listed in borg create --help)zdo not change anythingzprint statistics at endz--targetr ZTARGETzocreate a new archive with the name ARCHIVE, do not replace existing archive (only applies for a single archive))r� r r� r� r� )r� r� r� r r� z�manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). alternatively, give a reference file/directory.z --recompressrz rx � if-differentry aj recompress data chunks according to ``--compression``. MODE `if-different`: recompress if current compression is with a different compression algorithm (the level is not considered). MODE `always`: recompress even if current compression is with the same compression algorithm (use this to change the compression level). MODE `never` (default): do not recompress.)r r� r1 r� r r2 r� z�specify the chunker parameters (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE) or `default` to use the current defaults. default: %d,%d,%d,%dz!repository or archive to recreatez)paths to recreate; patterns are supporteda�
This command runs a user-specified command while the repository lock is held.
It will first try to acquire the lock (make sure that no other operation is
running in the repo), then execute the given command as a subprocess and wait
for its termination, release the lock and return the user command's return
code as borg's return code.
.. note::
If you copy a repository with the lock held, the lock will be present in
the copy. Thus, before using borg on the copy from a different host,
you need to use "borg break-lock" on the copied repository, because
Borg is cautious and does not automatically remove stale locks made by a different host.
z with-lockz run user command with lock heldzrepository to lockr� ZCOMMANDzcommand to runr~ ZARGSzcommand argumentsa
This command gets and sets options in a local repository or cache config file.
For security reasons, this command only works on local repositories.
To delete a config value entirely, use ``--delete``. To list the values
of the configuration file or the default values, use ``--list``. To get and existing
key, pass only the key name. To set a key, pass both the key name and
the new value. Keys can be specified in the format "section.name" or
simply "name"; the section will default to "repository" and "cache" for
the repo and cache configs, respectively.
By default, borg config manipulates the repository config file. Using ``--cache``
edits the repository cache's config file instead.
rr z get and set configuration valuesz--cacher� z&get and set values from the repo cachez--deletez#delete the key from the config filez-lz"list the configuration of the repor� )r� r� zrepository to configurezname of config keyr� ZVALUEznew value for keyr� z
Extra help)r/ r* r) z
--epilog-onlyr� )r� r� z --usage-onlyr� r� r zadditional help on TOPIC)r r� r1 r� al
These commands are not intended for normal use and potentially very
dangerous if used incorrectly.
They exist to improve debugging capabilities without direct system access, e.g.
in case you ever run into some severe malfunction. Use them only if you know
what you are doing or if a trusted developer tells you what to do.rc z/debugging command (not intended for normal use)z�
This command displays some system information that might be useful for bug
reports and debugging problems. If a traceback happens, this information is
already appended at the end of the traceback.
z5show system infos for debugging / bug reports (debug)zq
This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files.
zdump-archive-itemsz%dump archive items (metadata) (debug)zarchive to dumpz\
This command dumps all metadata of an archive in a decoded form to a file.
z dump-archivez%dump decoded archive metadata (debug)zfile to dump data intozc
This command dumps manifest metadata of a repository in a decoded form to a file.
z
dump-manifestz(dump decoded repository metadata (debug)zrepository to dumpz`
This command dumps raw (but decrypted and decompressed) repo objects to files.
zdump-repo-objszdump repo objects (debug)z--ghostr� zRdump all segment file contents, including deleted/uncommitted objects and commits.zx
This command searches raw (but decrypted and decompressed) repo objects for a specific bytes sequence.
zsearch-repo-objszsearch repo objects (debug)zrepository to searchr� ZWANTEDzCterm to search the repo for, either 0x1234abcd hex term or a stringzB
This command gets an object from the repository.
zget-objz"get object from repository (debug)zrepository to user ZIDz"hex object ID to get from the repoz file to write object data intoz@
This command puts objects into the repository.
zput-objz put object to repository (debug)�+z)file(s) to read and create object(s) fromzC
This command deletes objects from the repository.
z
delete-objz%delete object from repository (debug)r� ZIDsz(hex object ID(s) to delete from the repoz\
This command displays the reference count for objects from the repository.
z refcount-objz0show refcount for object from repository (debug)z&hex object ID(s) to show refcounts forz?
This command dumps the repository hints data.
z
dump-hintszdump repo hints (debug)zQ
Convert a Borg profile to a Python cProfile compatible profile.
zconvert-profilez.convert Borg profile to Python profile (debug)r� ZINPUTr� z Borg profiler� ZOUTPUTr� z Output filez%These commands do various benchmarks.Z benchmarkzbenchmark commandaV
This command benchmarks borg CRUD (create, read, update, delete) operations.
It creates input data below the given PATH and backups this data into the given REPO.
The REPO must already exist (it could be a fresh empty repo or an existing repo, the
command will create / read / update / delete some archives named borg-benchmark-crud\* there.
Make sure you have free space there, you'll need about 1GB each (+ overhead).
If your repository is encrypted and borg needs a passphrase to unlock the key, use::
BORG_PASSPHRASE=mysecret borg benchmark crud REPO PATH
Measurements are done with different input file sizes and counts.
The file contents are very artificial (either all zero or all random),
thus the measurement results do not necessarily reflect performance with real data.
Also, due to the kind of content used, no compression is used in these benchmarks.
C- == borg create (1st archive creation, no compression, do not use files cache)
C-Z- == all-zero files. full dedup, this is primarily measuring reader/chunker/hasher.
C-R- == random files. no dedup, measuring throughput through all processing stages.
R- == borg extract (extract archive, dry-run, do everything, but do not write files to disk)
R-Z- == all zero files. Measuring heavily duplicated files.
R-R- == random files. No duplication here, measuring throughput through all processing
stages, except writing to disk.
U- == borg create (2nd archive creation of unchanged input files, measure files cache speed)
The throughput value is kind of virtual here, it does not actually read the file.
U-Z- == needs to check the 2 all-zero chunks' existence in the repo.
U-R- == needs to check existence of a lot of different chunks in the repo.
D- == borg delete archive (delete last remaining archive, measure deletion + compaction)
D-Z- == few chunks to delete / few segments to compact/remove.
D-R- == many chunks to delete / many segments to compact/remove.
Please note that there might be quite some variance in these measurements.
Try multiple measurements and having a otherwise idle machine (and network, if you use it).
Zcrudz7benchmarks borg CRUD (create, extract, update, delete).z,repository to use for benchmark (must exist)z(path were to create benchmark input data)r+ r, r- )rx r5 ry )Wr� �ArgumentParserr� Z set_defaultsr� �partial�do_maincommand_helpr r� r� r r r> r
r) r� ZRawDescriptionHelpFormatterr0 r� Zadd_subparsersZ
add_parserr, r� r� r� r� r r r� r r r r r r( ra rb ZSTDIN_MODE_DEFAULTr r5 ZFILES_CACHE_MODE_UI_DEFAULTr2 rD r� r. ZCHUNKER_PARAMSr r+ r� r r- r" r* r6 Z keys_helprR rS r8 r7 rG rD rO r� rh rA rw r} r� Z REMAINDERr� r' r� r2 r� r� r� r� r� r� r� r� r� r� r� r� ZFileTyperR )7r� r r� r$ r( r� Z
common_parserZmid_common_parserZ mount_epilogr" Z
subparsersZ serve_epilogZ init_epilogZ check_epilogZ key_parsersZkey_export_epilogZkey_import_epilogZchange_passphrase_epilogZmigrate_to_repokey_epilogZ
create_epilogr# Zfs_groupZ
archive_groupZextract_epilogZexport_tar_epilogZ diff_epilogZ
rename_epilogZ
delete_epilogZ list_epilogZ
umount_epilogZ info_epilogZbreak_lock_epilogZ prune_epilogZupgrade_epilogZrecreate_epilogZwith_lock_epilogZ
config_epilogrv Z debug_epilogZ
debug_parsersZdebug_info_epilogZ debug_dump_archive_items_epilogZdebug_dump_archive_epilogZdebug_dump_manifest_epilogZdebug_dump_repo_objs_epilogZ debug_search_repo_objs_epilogZdebug_get_obj_epilogZdebug_put_obj_epilogZdebug_delete_obj_epilogZdebug_refcount_obj_epilogZdebug_dump_hints_epilogZ debug_convert_profile_epilogZbenchmark_epilogZbenchmark_parsersZbench_crud_epilogr )r! r r� � build_parser�
s�
/
6
7
!
"
"
1:
/
b
2
&
zArchiver.build_parserc C s� | j |dd� �}|dk r�|j| jkr�tj|�}ttjdd� |��}| j |dd� �}|j|jkr�ddddh}d d
d d h}t� }x@|D ]8} | |ks�t d
| ��t
|| |�}
|
|k r�t || |
� q�W |S )zWusually, just returns argv, except if we deal with a ssh forced command for borg serve.r Nc S s d| kS )N�=r )rl r r r� r� I s z#Archiver.get_args.<locals>.<lambda>r� r� r� r� r r� r r z*allowlist has denylisted attribute name %s) r) r� r� r� r� r� � itertools� dropwhiler� r� r{ r ) r� �argv�cmdr� Z client_argvZ
client_resultZdenylistZ allowlistZ not_presentZ attr_namer� r r r� �get_args> s*
zArchiver.get_argsc C s� |r| j |�}| j� }|j|p"dg�}|jj|� t|�}|| jkrV|j rV|jd� t |dd�s�| j
| j | j | j
| j| j| j| j| jh }||kr�td��t |dd �r�|jj|j�|_|S )Nz-hz Need at least one PATH argument.r� Tz:Not allowed to bypass locking mechanism for chosen commandrD )r� r: r) r� r r� r( rd r� r{ r r� r r� r+ rO rG r> rD r( r� Zwith_timestamprD )r� r~ r� r� Zbypass_allowedr r r� r) i s"
zArchiver.parse_argsc C s |s
t � t� t|� d S )N)rI rJ rw )r� r� �is_server r r� �
prerun_checks~ s zArchiver.prerun_checksc C sN dddddd�}x8|j � D ],\}}|j|d�}tj|�j|r@dnd � qW d
S ) zN turn on INFO level logging for args that imply that they will produce output zborg.output.listzborg.output.show-versionzborg.output.show-rczborg.output.statszborg.output.progress)r� r r rp r� F�INFOZWARNN)r� r� r� r� �setLevel)r� r~ Z
option_loggerZoptionZ logger_nameZ
option_setr r r� �_setup_implied_logging� s z Archiver._setup_implied_loggingc C s@ x:|j D ]0}d|kr d| }tjd|� tj|�jd� qW dS )z9Turn on DEBUG level logging for specified --debug-topics.r� z borg.debug.zEnabling debug topic %s�DEBUGN)r r� rc r� r� rD )r� r~ r� r r r� �_setup_topic_debugging� s
z Archiver._setup_topic_debuggingc C s� t j|j� |j| _t|�}|| jk}t|j||jd� |j| _| j|O _| j t
|�� | j |� t |dd�r�t |dd�r�t
jd� d|_|jr�tjd�jdt � | jt
|� t� s�t
jd� t
jd � t
jd
� tt�S t� r�t
jt� |j �r�d d l }d d l }t
j d |j � t |j d
��p}|j!� }t"t#� �}|j$� z t||��S |j%� |j&� |j j'd��rz|j(|j|� nt)j*|j|dd� X W d Q R X n t||��S d S )N)�levelrA r� rp FrS z;Ignoring --stats. It is not supported when using --dry-run.zborg.output.show-versionzborgbackup version %szLYou do not have a supported msgpack[-python] version installed. Terminating.zVThis should never happen as specific, supported versions are required by our setup.py.z-Do not contact borgbackup support about this.r z Writing execution profile to %sr� z.pyprofT)Z use_bin_type)+r r r� r� r� r
r r� r� rE �varsrG r{ r� r� rp r r� r� r� r rB rM r� r* r& rL �PURE_PYTHON_MSGPACK_WARNINGr �cProfiler� rc r� ZProfiler� �locals�enable�disableZsnapshot_statsr� r� r_ �pack) r� r~ r� rA rK r� r> Zprofiler� variablesr r r� �run� sL
z Archiver.run)NN)NN)FFN)ri rj rk )NN)N)Sr� r r r� r� r� r� � staticmethodr� r� r� r� r� r rF r( ro r r r r r rR r� r( rk r� ZREADr� r+ r� r� r r" r* r$ r% r> r< rD rG rE rF rO rM rN r) rh rw r} r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r� rR rS r� r� r9 r� r r: r@ r) rB rE rG rQ r r r r� r� � s�
'
O `
ZS w
F80 P J + x 2 8C Z
9
8
~ `+
r� c C s� t | tj��� x�tj|�D ]�}|d |d j }}|d
kr�|d }y|d j� }|d j}W n tk
rx d\}}Y nX t j
dj |t |�t |��� P |dkr|d
j
}y|d j� }W n tk
r� d}Y nX t j
d j |t |��� P qW W d Q R X d S )zLsearch the stack for infos about the currently processed file and print them� r r� rk r� r> rX z {0} {1}/{2}r� r} z {0} {1}/???N)r� rk )r r )r� )rQ �signal�SIG_IGN�inspectZgetouterframes�f_locals�tell�st_sizer� r� r� r r: r� )�sig_no�stack�framer� �locr� �posr� r r r� �sig_info_handler� s(
r_ c C s* t dtj� jdd� tjd� tj� d S )Nz)
Received SIGUSR2 at %s, dumping trace...r )Z microsecond)r� )r� r Znowr� r� r� � faulthandlerZdump_traceback)rZ r[ r r r� �sig_trace_handler� s ra c J C s@ t tjjtjjddd�t_t tjjtjjddd�t_tj� tdt t
����� tdt t ����� tdt t ����� tdt
���� tdt���� td t
���j t� } d } }}tj}y| jtjtjjd
��}W n� tk
�rR } zf|j� }|jr�tjntj}d tj� t � f }t |tjd � |tjk�r6t |tjd � tj |j � W Y d d }~X nX y| j |�}W �n� tk
�r� } zD|j� }t!|�j"}|j�r�tjntj}d tj� t � f }|j }W Y d d }~X �nb t#j$k
�rX } zr|j%d$k�o�|j}|j%}|�r tjntj}|�r|j&}n|j� }dj'dd� |j j(� D ��}|dt � 7 }t)}W Y d d }~X n� t*k
�r� d}d}tj}d tj� t � f }t)}Y n� t
k
�r� d}tj}d tj� t � f }t+d }Y nb t k
�r d}d}tj}d tj� t � f }t+d }Y n& t k
�r( d}d}t+d }Y nX |�r>t,j-||d� |�rPt,j.||� |j/�r�tj0d �} d }
|t1k�r�| j2|
d |f � nr|t3k�r�| j4|
d |f � nT|t)k�r�| j-|
d |f � n6|t+k�r�| j-|
d!|f � n| j-|
d"|�p�d#f � tj |� W d Q R X W d Q R X W d Q R X W d Q R X W d Q R X W d Q R X d S )%Nr� T)�line_buffering�SIGINT�SIGHUP�SIGTERM�SIGUSR1�SIGUSR2ZSIGINFOZSSH_ORIGINAL_COMMANDz%s
%s)r� � LockTimeoutr2 c s s | ]}d | V qdS )z
Borg server: Nr )r| r r r r� r� % s zmain.<locals>.<genexpr>zLocal Exceptionr� zKeyboard interruptrz zReceived SIGTERMzSignal.SIGTERM� zReceived SIGHUP.z
Signal.SIGHUPr )r� zborg.output.show-rcz!terminating with %s status, rc %d�successr� r� rT Zabnormali� )rh )5rU r� r� �bufferr� r� r` rM rQ rR r� rS rT r_ ra r� r� ZERRORr@ r> r rQ r� r( Z get_message� tracebackrF �
format_excrO r� �exitr� rQ r� r rp ZRPCErrorZexception_classZexception_fullr4 r r& r� r' r� r� �logr r� r$ r� r% r� ) Zarchiverr� r� �tbZ tb_log_levelr~ rw r� Z importantZ rc_loggerZexit_msgr r r� �main� s�
rq �__main__) FFFTFTFTN)�r� rl r� r r� r` r� r� rV r< r� r� r re r� r8 rT r] r� r� rR r& Zbinasciir r �
contextlibr r r r r� r r
r= r� r r r
Zalgorithms.checksumsr r� r r r r r r r r r r� r r r Z constants�compressr r� r r r r r! r" Zcrypto.keymanagerr# r$ r% r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rC rD rE rF rG rH rI rJ rK rL rM rN rO rP rQ rR rS rT rU rV rW rX rY rZ r[ r\ r] r^ r_ r` ra rb rc Znanorstrd r� re rf rg rh ri r} rj rk r� rl rm rn Zremotero rp rq r� rr rs rt ru rv rw Zupgraderrx ry �
BaseException� print_excrn r� rr rJ r� r� r� r� r� r� r_ ra rq r� r r r r� �<module> s
I !_