Current File : //usr/lib64/python3.6/site-packages/borg/__pycache__/archive.cpython-36.pyc
3

up�d	h�@s<ddlZddlZddlZddlZddlZddlZddlZddlmZddl	m	Z	m
Z
mZddlm
Z
ddlmZddlmZddlmZddlmZd	d
lmZe�Zd	dlmZd	dlmZd	d
lmZd	dlmZm Z d	dl!m"Z"m#Z#d	dl$Td	dl%m&Z&m'Z'm(Z(d	dl)m*Z*d	dl)m+Z+d	dl)m,Z,m-Z-d	dl)m.Z.m/Z/m0Z0d	dl)m1Z1m2Z2m3Z3m4Z4d	dl)m5Z5m6Z6d	dl)m7Z7m8Z8m9Z9m:Z:m;Z;d	dl)m<Z<m=Z=m>Z>m?Z?d	dl)m@Z@d	dl)mAZAd	dl)mBZBd	dl)mCZCmDZDmEZEd	dl)mFZFd	dl)mGZGd	d lHmIZImJZJmKZKd	d!lLmMZMmNZNd	d"lOmPZPmQZQmRZRmSZSmTZTmUZUd	d#lVmWZWd	d$lXmYZYmZZZe[ed%�Z\ej]e^ed&d�BZ_e_e^ed'd�BZ`Gd(d)�d)�Zad*d+�ZbGd,d-�d-ec�ZdGd.d/�d/ec�ZeGd0d1�d1�Zfef�Zgd2d3�ZhGd4d5�d5�ZiGd6d7�d7�ZjGd8d9�d9ej�ZkGd:d;�d;�Zld<d=�ZmGd>d?�d?�ZnGd@dA�dA�ZoGdBdC�dC�ZpdS)D�N)�contextmanager)�datetime�timezone�	timedelta)�partial)�getuser)�BytesIO)�groupby)�get_terminal_size�)�
create_logger)�xattr)�Chunker)�ChunkListEntry)�key_factory�UnsupportedPayloadError)�
Compressor�CompressionSpec)�*)�
ChunkIndex�ChunkIndexEntry�CacheSynchronizer)�Manifest)�hardlinkable)�ChunkIteratorFileWrapper�	open_item)�Error�IntegrityError�set_ec)�uid2user�user2uid�	gid2group�	group2gid)�parse_timestamp�to_localtime)�OutputTimestamp�format_timedelta�format_file_size�file_status�FileSize)�safe_encode�safe_decode�make_path_safe�remove_surrogates)�
StableDict)�
bin_to_hex)�safe_ns)�ellipsis_truncate�ProgressIndicatorPercent�	log_multi)�msgpack)�workarounds)�PathPrefixPattern�FnmatchPattern�	IECommand)�Item�ArchiveItem)�acl_get�acl_set�	set_flags�	get_flags�swidth�hostname)�cache_if_remote)�
Repository�LIST_SCAN_LIMIT�linkZO_BINARY�	O_NOATIMEc@sheZdZddd�Zdd�ZdZdd�Zd	d
�Zdd�Ze	d
d��Z
e	dd��Ze	dd��Zddd�Z
dS)�
StatisticsFcCs(||_d|_|_|_|_d|_dS)Nr)�output_json�osize�csize�usize�nfiles�
last_progress)�selfrG�rN�/usr/lib64/python3.6/archive.py�__init__7szStatistics.__init__cCs2|j|7_|j|7_|r.|j|7_dS)N)rHrIrJ)rM�sizerI�uniquerNrNrO�update<szStatistics.updatezO{label:15} {stats.osize_fmt:>20s} {stats.csize_fmt:>20s} {stats.usize_fmt:>20s}cCs|jj|dd�S)Nz
This archive:)�statsZlabel)�summary�format)rMrNrNrO�__str__DszStatistics.__str__cCsdjt|�jt|�|d�S)NzF<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>)�cls�hashrM)rV�type�__name__�id)rMrNrNrO�__repr__GszStatistics.__repr__cCs"t|j�t|j�t|j�|jd�S)N)Z
original_sizeZcompressed_sizeZdeduplicated_sizerK)r)rHrIrJrK)rMrNrNrO�as_dictKszStatistics.as_dictcCs
t|j�S)N)r'rH)rMrNrNrO�	osize_fmtSszStatistics.osize_fmtcCs
t|j�S)N)r'rJ)rMrNrNrO�	usize_fmtWszStatistics.usize_fmtcCs
t|j�S)N)r'rI)rMrNrNrO�	csize_fmt[szStatistics.csize_fmtNc
Cstj�}|dks ||j|k�r||_|jrz|sP|j�}t|rD|jnd�|d<ni}|jtj�d|d��tj	|�}d}nrt
�\}	}
|s�dj|�}|r�t|j�nd}|	t|�}|dkr�d}|	t|�}|dkr�|t
||�7}nd	|	}d
}t|||p�tjdd�dS)
N��pathZarchive_progress)�timerZZfinished�
z={0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N ��� �
T)�end�file�flush)rd�	monotonicrLrGr^r-rcrS�json�dumpsr
rVr?r1�print�sys�stderr)
rM�item�final�stream�dtZnow�data�msgrj�columns�linesrcZspacerNrNrO�
show_progress_s6



zStatistics.show_progress)F)NFNN)r[�
__module__�__qualname__rPrSrUrWr]r^�propertyr_r`rar{rNrNrNrOrF5s
rFcCstj|�ptj|�ptj|�S)N)�stat�S_ISBLK�S_ISCHR�S_ISFIFO)�moderNrNrO�
is_special�sr�c@seZdZdZdS)�BackupErrorzY
    Exception raised for non-OSError-based exceptions while accessing backup files.
    N)r[r|r}�__doc__rNrNrNrOr��sr�c@s eZdZdZdd�Zdd�ZdS)�
BackupOSErrora�
    Wrapper for OSError raised while accessing backup files.

    Borg does different kinds of IO, and IO failures have different consequences.
    This wrapper represents failures of input file or extraction IO.
    These are non-critical and are only reported (exit code = 1, warning).

    Any unwrapped IO error is critical and aborts execution (for example repository IO failure).
    cCs(||_||_|j|_|j|_|j|_dS)N)�op�os_error�errno�strerror�filename)rMr�r�rNrNrOrP�s
zBackupOSError.__init__cCs$|jrd|j|jfSt|j�SdS)Nz%s: %s)r�r��str)rMrNrNrOrW�szBackupOSError.__str__N)r[r|r}r�rPrWrNrNrNrOr��s	r�c@s*eZdZdZd	dd�Zdd�Zdd�ZdS)
�BackupIOrbcCs
||_|S)N)r�)rMr�rNrNrO�__call__�szBackupIO.__call__cCsdS)NrN)rMrNrNrO�	__enter__�szBackupIO.__enter__cCs |rt|t�rt|j|�|�dS)N)�
issubclass�OSErrorr�r�)rM�exc_typeZexc_valZexc_tbrNrNrO�__exit__�szBackupIO.__exit__N)rb)r[r|r}r�r�r�r�rNrNrNrOr��s
r�ccsHdt_x<t�(yt|�}Wntk
r.dSXWdQRX|VqWdS)N�read)�	backup_ior��next�
StopIteration)�iteratorrsrNrNrO�backup_io_iter�sr�c@s(eZdZdd�Zd	dd�Zd
dd�ZdS)�DownloadPipelinecCs||_||_dS)N)�
repository�key)rMr�r�rNrNrOrP�szDownloadPipeline.__init__NFc#sb�fdd�}t�}tjdd�}�x<�j|�D�],}	|j|	�dd�|D�}
x&|
D]}d|krPdd�|jD�|_qPW�r��fd	d�|
D�}
|�rD�o�|�r x�|
D]|}t|j�r�|jd
�}|dkr�d|kr�||j�|jdd
�r�|j	|j
�q�||kr�||\}
}|
dk	�r||
�|j	|�q�Wn$x"|
D]}d|k�r&||j��q&Wx|
D]}|V�qJWq,WdS)a�
        Return iterator of items.

        *ids* is a chunk ID list of an item stream. *filter* is a callable
        to decide whether an item will be yielded. *preload* preloads the data chunks of every yielded item.

        Warning: if *preload* is True then all data chunks of every yielded item have to be retrieved,
        otherwise preloaded chunks will accumulate in RemoteRepository and create a memory leak.
        cs�jjdd�|D��dS)NcSsg|]
}|j�qSrN)r\)�.0�crNrNrO�
<listcomp>�szBDownloadPipeline.unpack_many.<locals>._preload.<locals>.<listcomp>)r��preload)�chunks)rMrNrO�_preload�sz.DownloadPipeline.unpack_many.<locals>._preloadF)�use_listcSsg|]}t|d��qS))�
internal_dict)r9)r�rsrNrNrOr��sz0DownloadPipeline.unpack_many.<locals>.<listcomp>r�cSsg|]}t|��qSrN)r)r��erNrNrOr��scsg|]}�|�r|�qSrNrN)r�rs)�filterrNrOr��s�sourceN�hardlink_masterT)�setr4�Unpacker�
fetch_many�feedr�rr��get�addrc)rM�idsr��partial_extractr��hardlink_mastersr�Zmasters_preloaded�unpackerrw�itemsrsr�r��_rN)r�rMrO�unpack_many�s>











zDownloadPipeline.unpack_manyccs8x2t||jj||d��D]\}}|jj||�VqWdS)N)�is_preloaded)�zipr��get_manyr��decrypt)rMr�r��id_rwrNrNrOr�s zDownloadPipeline.fetch_many)NFFN)F)r[r|r}rPr�r�rNrNrNrOr��s
9r�c@s>eZdZdZefdd�Zdd�Zdd�Zdd
d�Zdd
�Z	dS)�ChunkBufferrgicCs:t�|_tjdd�|_g|_||_t|jjf|��|_	dS)N�surrogateescape)Zunicode_errors)
r�bufferr4ZPacker�packerr�r�r�
chunk_seed�chunker)rMr��chunker_paramsrNrNrOrPs
zChunkBuffer.__init__cCs,|jj|jj|j���|j�r(|j�dS)N)r��writer��packr^�is_fullrl)rMrsrNrNrOr�szChunkBuffer.addcCst�dS)N)�NotImplementedError)rM�chunkrNrNrO�write_chunkszChunkBuffer.write_chunkFcCs�|jj�dkrdS|jjd�tdd�|jj|j�D��}|jjd�|jjd�|sbt|�dkrfdnd}x&|d|�D]}|jj	|j
|��qxW|dkr�|jj|d�dS)Nrcss|]}t|�VqdS)N)�bytes)r��srNrNrO�	<genexpr>sz$ChunkBuffer.flush.<locals>.<genexpr>r���r�r�)r��tell�seek�listr��chunkify�truncate�lenr��appendr�r�)rMrlr�rjr�rNrNrOrlszChunkBuffer.flushcCs|jj�|jkS)N)r�r��BUFFER_SIZE)rMrNrNrOr�'szChunkBuffer.is_fullNi i�)F)
r[r|r}r��ITEMS_CHUNKER_PARAMSrPr�r�rlr�rNrNrNrOr�s
r�cs(eZdZef�fdd�	Zdd�Z�ZS)�CacheChunkBuffercst�j||�||_||_dS)N)�superrP�cacherT)rMr�r�rTr�)�	__class__rNrOrP-szCacheChunkBuffer.__init__cCs8|jj|jj|�||jdd�\}}}|jjjdd�|S)NF)�wait)r��	add_chunkr��id_hashrTr��async_response)rMr�r�r�rNrNrOr�2s$zCacheChunkBuffer.write_chunk)r[r|r}r�rPr��
__classcell__rNrN)r�rOr�+sr�c@s�eZdZGdd�de�ZGdd�de�ZGdd�de�Zddd	ddddddddedddddfd
d�Zdd
�Z	dd�Z
edd��Zedd��Z
edd��Zedd��Zedd��Zdd�Zdd�Zdd�ZdVd d!�ZdWd"d#�ZdXd%d&�Zd'd(�ZdYd)d*�Zd+d,�Zed-d.��ZdZd0d1�Zd[d2d3�Zd4d5�Zd6d7�Zd\d8d9�Z d:d;�Z!d<d=�Z"d>d?�Z#ed]d@dA��Z$dBdC�Z%dDdE�Z&dFdG�Z'dHdI�Z(dJdK�Z)d^dLdM�Z*dNdO�Z+dPdQ�Z,e-d_dRdS��Z.e-dTdU��Z/dS)`�Archivec@seZdZdZdS)zArchive.DoesNotExistzArchive {} does not existN)r[r|r}r�rNrNrNrO�DoesNotExist:sr�c@seZdZdZdS)zArchive.AlreadyExistszArchive {} already existsN)r[r|r}r�rNrNrNrO�
AlreadyExists=sr�c@seZdZdZdS)z+Archive.IncompatibleFilesystemEncodingErrorzrFailed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.N)r[r|r}r�rNrNrNrO�#IncompatibleFilesystemEncodingError@sr�NFicCs�tj�|_||_||_||_||_i|_t|d�|_	||_
||_||_d|_
d|_||_||_|	|_|
|_||_||_|
|_||_|dk|dkks�td��|dkr�tj�}tj�}||_||_||_|dkr�tj�}||_||_ t!|j|j�|_"||_#|j#�r�t$|j|j|j	�|_%t&|jj'f|��|_(||j)k�r<|j*|��tj�|_+d}xrd||�r`d|�pbdf|_,|j,|j)k�rzP|d7}�qLWn4|jj)j-|�}|dk�r�|j.|��|j/|j0�d|_1dS)	N)rGFzULogic error: if start is given, start_monotonic must be given as well and vice versa.rz%s.checkpoint%sz.%drbr)2�os�getcwd�cwdr�r�r��manifest�
hard_linksrFrTr{�nameZname_in_manifest�comment�tam_verified�checkpoint_interval�
numeric_owner�noatime�noctime�nobirthtime�
nobsdflags�noacls�noxattrs�AssertionErrorr�utcnowrdrmr��start�start_monotonicrj�consider_part_filesr��pipeline�creater��items_bufferrr�r��archivesr��last_checkpoint�checkpoint_namer�r��loadr\�zeros)rMr�r�r�r�r�r�r�r�r�r�r�r�r�r��progressr�r�r�rjr�Zlog_json�i�inforNrNrOrPCsd




zArchive.__init__cCsN|jj||jj|��}|jj|dd�\}|_}t|d�}|jdkrJtd��|S)NT)�force_tam_not_required)r�rz Unknown archive metadata version)	r�r�r�r��unpack_and_verify_archiver�r:�version�	Exception)rMr\rw�archiver��metadatarNrNrO�
_load_meta|s

zArchive._load_metacCsH||_|j|j�|_dd�|jjD�|j_|jj|_|jjdd�|_dS)NcSsg|]}t|��qSrN)r+)r��argrNrNrOr��sz Archive.load.<locals>.<listcomp>r�rb)r\r	r�cmdliner�r�r�)rMr\rNrNrOr��s

zArchive.loadcCs|jj}t|�S)z,Timestamp of archive creation (start) in UTC)rrdr#)rM�tsrNrNrOr�sz
Archive.tscCs|jjd�p|jj}t|�S)z*Timestamp of archive creation (end) in UTC�time_end)rr�rdr#)rMrrNrNrO�ts_end�szArchive.ts_endcCs
t|j�S)N)r/r\)rMrNrNrO�fpr�szArchive.fprcCst|j|j�S)N)r&rjr�)rMrNrNrO�duration�szArchive.durationcCst|j|j�S)N)r&rr)rMrNrNrO�duration_from_meta�szArchive.duration_from_metac	Cs�|jr.|j}|jjtjd�}|jjtjd�}n|j|j�}|j	}|j
}|j|jt
|�t
|�||j�|j�d|jj|jjtid�}|jr�tj|d<n6|j|jj|jj|jj|jjdd�|jjdd�d��|S)	N)�tzinfoZmax_archive_size)r�r\r�rjrrTZlimits�command_liner�rbr�)rr@�usernamer�r�)r�rTr��replacer�utcrj�
calc_statsr�rrr�rr%Z
total_secondsr^r�r\rI�
MAX_DATA_SIZErq�argvrSrrr@rr�)rMrTr�rjrrNrNrOr�s.
zArchive.infocCsBdj|t|jjtjd��t|jjtjd��|jj|j	j
td�S)Nz�Archive name: {0.name}
Archive fingerprint: {0.fpr}
Time (start): {start}
Time (end):   {end}
Duration: {0.duration}
Number of files: {0.stats.nfiles}
Utilization of max. archive size: {csize_max:.0%}
)r)r�rjZ	csize_max)rVr%r�rrrrjr�r�r\rIr)rMrNrNrOrW�s
	zArchive.__str__cCs
d|jS)NzArchive(%r))r�)rMrNrNrOr]�szArchive.__repr__cCs$|jrd|krdS|r ||�SdS)N�partFT)r�)rMrsr�rNrNrO�item_filter�szArchive.item_filterc#sR�o
|o
|s|dk	st�x2�jj�jj|||��fdd�d�D]
}|Vq@WdS)Ncs�j|��S)N)r)rs)r�rMrNrO�<lambda>�sz$Archive.iter_items.<locals>.<lambda>)r�r�r�r�)r�r�r�rr�)rMr�r�r�r�rsrN)r�rMrO�
iter_items�s
zArchive.iter_itemsTcCs*|r|jr|jj|dd�|jj|�dS)Ng�������?)rsrv)r{rTr�r�)rMrsr{rNrNrO�add_item�s
zArchive.add_itemcCs.|j|j�|jj|j=|jj|j|j�dS)N)�saver�r�r�r��chunk_decrefr\rT)rMrNrNrO�write_checkpoint�szArchive.write_checkpointcCs�|p|j}||jjkr |j|��|jjdd�ttj�|j	d�}|dkr\t
j�}||}n||}|}||_||_
d||p~d|jjtjtt�|jt�|jt�|jd�
}|j|p�i�t|�}|jj|j�dd�}	|jj|	�|_y|jj|j|	|j�WnFt k
�r>}
z(t!|
�}d	|k�r,t"d
|��n�WYdd}
~
XnXx|j#j$dd�dk	�rZ�qBW|j|jf|jj|<|jj%�|j#j&�|jj&�dS)NT)rl)Zsecondsrrb)
rr�r�r�rr@rrdr
r�sarchive)�contextzMore than allowed put dataz#%s - archive too big (issue #1473)!)r�)'r�r�r�r�r�rlrrdrmr�rr�r�rjr�rqrr@rZstrftimeZ
ISO_FORMATr�rSr:r��pack_and_authenticate_metadatar^r�r\r�r�rTrr�rr�r�r��commit)rMr�r��	timestamp�additional_metadatarrjr�rrw�errZerr_msgrNrNrOr�sN






zArchive.savec
s��fdd�}t��t��}||j�tt|jj�ddd�}xRt|jj|jj	|jj��D]4\}}|j
dd�||�|jj||�}|j
|�qXW�j�j�d}|j�t�}	|jr�|jn
|j|j|	_|jr�|jn
|j|j|	_|jr�|jn
|j|j|	_||	_|	S)	Ncs"�j|}�j|d|j|j�dS)Nr)r�r�rQrI)r\�entry)�
archive_indexr�rNrOr�s
zArchive.calc_stats.<locals>.addzCalculating statistics... %3d%%zarchive.calc_stats)�totalrx�msgidr)�increase�)rrr\r2r�rr�r�r�r��showr�r�r�Z
stats_againstr��finishrFr�Znum_files_totalsZnum_files_partsrKZsize_totalsZ
size_partsrHZcsize_totalsZcsize_partsrIrJ)
rMr�r��sync�pir\r�rwZunique_csizerTrN)r)r�rOrs*
$zArchive.calc_statsccs�d}d|kr�tjj|f|jjtj�|d���}|j|jd|f�\}	}
|
rttrttd��tj	|
|�d}WdQRXn|	dk	r�|	|_
|V|r�|r�tr�d|f||jd�p�|<ndS)NFr�rDT)r�rc�joinr��split�sepr��has_linkr�rDr�)rM�destrsrc�stripped_components�
original_pathr��hardlink_setr�r�Zlink_targetrNrNrO�extract_helper8s$

zArchive.extract_helperrc
?Cs�|pi}d|k}
|s|r�d|kr�d}x`|jjdd�|jD�dd�D]@}|	rd|	jt|�t|j�gd�|rvtjj	j
|�|t|�7}qBW|r�tjj	j�d	|kr�|j}
|
|kr�t
d
j|
|���|
r�t
d��dS|p�|j}|j}|jjd�r�td��tjj||j�}y6tj|dd�}tj|j��r*tj|�n
tj|�Wn<tk
�r^|j|tj��d�Yntk
�rrYnXdd�}|j}tj|��r<td��||�WdQRX|j ||||||���r}|�r�dS|�r�|j!dk�r�dd|j"d>|_!td��t#|d�}WdQRX|��dd�|jD�}x~|jj|dd�D]j}|	�r\|	jt|�t|j�gd�td��6|�r�|j!j|��r�|j$t|�d�n
|j
|�WdQRX�q8Wtd��8|j%�}}|j&|�|j�|j'|||j(�d�WdQRXWdQRXd	|k�r |j}
|
|k�r t
d
j|
|���|
�r.t
d��WdQRXdSt��|tj|��r�||�tjj)|��sptj*|�|�r�|j'||��n4tj+|��r�||�|j,}ytj-||�Wn(tk
�r�|j|tj��d�YnX|j'||dd�n�tj.|��rB||�|j ||||||��&}|�r dStj/|�|j'||�WdQRXnxtj0|��sZtj1|��r�||�|j ||||||��.}|�r�dStj2||j|j3�|j'||�WdQRXntd|j��WdQRXdS) a�
        Extract archive item.

        :param item: the item to extract
        :param restore_attrs: restore file attributes
        :param dry_run: do not write any data
        :param stdout: write extracted data to stdout
        :param sparse: write sparse files (chunk-granularity, independent of the original being sparse)
        :param hardlink_masters: maps paths to (chunks, link_target) for extracting subtrees with hardlinks correctly
        :param stripped_components: stripped leading path components to correct hard link extraction
        :param original_path: 'path' key as stored in archive
        :param pi: ProgressIndicatorPercent (or similar) for file extraction progress (in bytes)
        �chunks_healthyr�rcSsg|]
}|j�qSrN)r\)r�r�rNrNrOr�fsz(Archive.extract_item.<locals>.<listcomp>T)r�)r,rrQz4Size inconsistency detected: size {}, chunks size {}zDFile has damaged (all-zero) chunks. Try running borg check --repair.N�/�../z!Path should be relative and localF)�follow_symlinkscSs&tjj|�}tjj|�s"tj|�dS)N)r�rc�dirname�exists�makedirs)rcZ
parent_dirrNrNrO�make_parent�sz)Archive.extract_item.<locals>.make_parentrA�r�open�wbcSsg|]
}|j�qSrN)r\)r�r�rNrNrOr��sr�Ztruncate_and_attrs)�fd)�symlinkzUnknown archive item type %r)r<r=)4r�r�r�r.r�r-rcrq�stdoutr�r�rlrQr�rVr��
startswithrr�r2r�S_ISDIR�st_mode�rmdir�unlink�UnicodeEncodeErrorr��getfilesystemencodingr�r��S_ISREGr�r:r�r�rDr�r�r��
restore_attrs�filenor@�mkdir�S_ISLNKr�rGr��mkfifor�r��mknod�rdev)rMrsrQ�dry_runrHZsparser�r7r8r1Zhas_damaged_chunks�item_chunks_sizerw�	item_sizer6rc�strBr�r9rFr��posr�rNrNrO�extract_itemRs�"







(




zArchive.extract_itemc2Cs�dt_d}}|js(t|j�}t|j�}|dkr6|jn|}|dkrH|jn|}y*|rbt	j
|||�nt	j|||dd�Wntk
r�YnX|r�t	j
||j�nByt	j||jdd�Wn*ttfk
r�|s�t	j||j�YnX|j}d|kr�|j}n|}d|k�rZ|j}	y6|�r*t	j|d||	fd�nt	j|d||	fdd�Wntk
�rXYnXy6|�rxt	j|d||fd�nt	j|d||fdd�Wntk
�r�YnX|j�s�t|||j�|j�s�|jd	i�}
x�|
j�D]�\}}ytj|�p�|||dd�Wn�tk
�r�}
zt|
jtjk�r(d
}n>|
jtjk�r<d}n*|
jtj k�rZdt!|�f}nt	j"|
j�}t#j$d
||j%�|�t&t'�WYdd}
~
XnX�q�W|j(�r�d|k�r�yt)||j*|d�Wntk
�r�YnXdS)zv
        Restore filesystem attributes on *path* (*fd*) from *item*.

        Does not access the repository.
        �attrsNF)r>�atime�	birthtime)�ns)rar>�xattrsztoo big for this filesystemz'xattrs not supported on this filesystemz(no space left on device [xattr len = %d]z*%s: when setting extended attribute %s: %s�bsdflags)rF)+r�r�r�r �userr"�group�uid�gidr��fchown�chownr��fchmodr��chmodr��SystemError�mtimer_r`�utimer�r<r�r�r�r
�setxattrr�ZE2BIGZENOTSUPZENOSPCr�r��logger�warning�decoderZEXIT_WARNINGr�r=rc)rMrcrsrGrFrfrgrmr_r`rb�k�vr�Zerr_strrNrNrOrQ�sx


 zArchive.restore_attrscCsz|j|j�}t|||�|jj|j�dd�}|jj|�}|jj|||j	�||j
f|jj|j
<|jj|j|j	�||_dS)Nsarchive)r")r	r\�setattrr�r#r^r�r�r�rTrdr�r�r�r )rMr��valuerrwZnew_idrNrNrO�set_meta2szArchive.set_metacCs<||jjkr|j|��|j}||_|jd|�|jj|=dS)Nr�)r�r�r�r�rw)rMr�ZoldnamerNrNrO�rename<s
zArchive.renamecs�Gdd�dt��t��d����fdd�	����fdd�}d��ytjdd	�}�jj}tt|�d
dd�}x�tt	|�j
j|���D]�\}\}	}
|r�|j|��j
j|	|
�}
|j|
�||	|�yDx>|D]6}t|d
�}d|kr�x|jD]\}}
}|||�q�Wq�WWq�ttfk
�r*�dk�r"�d�Yq�Xq�W|�r>|j�Wn.tjtjfk
�rn�dk�rf�d�YnX|�j|��jj�j=x�dd�dk	�r��q�W��r�tjd�tjd�dS)Nc@seZdZdZdS)z(Archive.delete.<locals>.ChunksIndexErrorzUChunk ID {} missing from chunks index, corrupted chunks index - aborting transaction.N)r[r|r}r�rNrNrNrO�ChunksIndexErrorEsryTcs8y�jj|d�Stjk
r2�dkr*�d��SXdS)N)r�rT)r�r�rB�ObjectNotFound)r�)�error�exception_ignored�forcedrMrNrO�fetch_async_responseJsz,Archive.delete.<locals>.fetch_async_responsecsJy�jj||dd�Wn$tk
r:t|�}�|��YnX�dd�dS)NF)r�)r�r �KeyErrorr/)r\rT�cid)ryr~rMrNrOr Usz$Archive.delete.<locals>.chunk_decrefF)r�zDecrementing references %3.0f%%zarchive.delete)r*rxr+)r�r�r)r�zAforced deletion succeeded, but the deleted archive was corrupted.z2borg check --repair is required to free all space.)T)r�objectr4r�rr�r2r��	enumerater�r�r�r.r�r�r�r9r��	TypeError�
ValueErrorr/ZUnpackExceptionrBrzr\r�r�r�rprq)rMrTrr}r r�Z	items_idsr1rZitems_idrwrs�chunk_idrQrIrN)ryr{r|r~r}rMrO�deleteDsL	$








zArchive.deletecCs�t|j|j|jt|j�d�}|js0t|j�|d<|jsDt|j	�|d<|j
rlt|d�rltt|j
d
��|d<|jr�d|d<|d	<nt|j�|d<t|j�|d	<|S)N)r�rfrgrmr_�ctime�st_birthtime�
�	r`rdreiʚ;)�dictrK�st_uid�st_gidr0�st_mtime_nsr��st_atime_nsr��st_ctime_nsr��hasattr�intr�r�rr!)rMr[r^rNrNrO�stat_simple_attrs�s zArchive.stat_simple_attrscCszi}td��H|jrintj|dd�}|jr0dnt||�}|jsPt||||j�WdQRX|rjt	|�|d<|rv||d<|S)Nz
extended statF)r>rrbrc)
r�r�r
Zget_allr�r>r�r;r�r.)rMr[rcr^rbrcrNrNrO�stat_ext_attrs�s
zArchive.stat_ext_attrscCs |j|�}|j|j||��|S)N)r�rSr�)rMr[rcr^rNrNrO�
stat_attrs�s
zArchive.stat_attrsc
cs�t|�}t|d�}d}|o"|jdk}|rT|jj|j|jf�}	|	dk	rP|	|_d}nd}||||fV|j|�|r�||j|j|jf<dS)N)rcFr�hT)	r,r9�st_nlinkr�r��st_ino�st_devr�r)
rMrcr[�statusrZ	safe_pathrsr��
hardlinkedr�rNrNrO�
create_helper�s

zArchive.create_helpercCs<|j||ddd�� \}}}}|j|j||��|SQRXdS)N�dF)r)r�rSr�)rMrcr[rsr�r�r�rNrNrO�process_dir�szArchive.process_dircCs8|j||d�� \}}}}|j|j||��|SQRXdS)N�f)r�rSr�)rMrcr[rsr�r�r�rNrNrO�process_fifo�szArchive.process_fifocCs@|j|||��(\}}}}|j|_|j|j||��|SQRXdS)N)r��st_rdevrWrSr�)rMrcr[Zdev_typersr�r�r�rNrNrO�process_dev�szArchive.process_devcCs`|j||ddd��D\}}}}td��tj|�}WdQRX||_|j|j||��|SQRXdS)Nr�F)r�readlink)r�r�r�r�r�rSr�)rMrcr[rsr�r�r�r�rNrNrO�process_symlink�s
zArchive.process_symlinkcCstt|j�d�}t|j�}|j|d�|_|jddd�|jd|7_||_|d7}|j|dd�|j�||fS)N)r�T)�memorize�from_chunksz
.borg_part_%drF)r{)	r9r^r�r��get_sizercrrr!)rMrs�
from_chunkZnumberZlengthrNrNrO�write_part_file�s
zArchive.write_part_filecs|s���fdd�}g|_t�dd�r2d|kr2|`d}d}x�|D]`}|jj||���jrj�jj|dd	��jr@tj��j	�jkr@�j
|||�\}}tj��_	q@W|dk�r|j|d�r؈j
|||�\}}tj��_	t�}	x"|jD]}
�j|
j
|	|
jd
�q�WdS)Ncs.�j�jj|�|�dd�}�jjjdd�|S)NF)r�)r�r�r�r�r�r�)rw�chunk_entry)r�rMrTrNrO�chunk_processor�sz+Archive.chunk_file.<locals>.chunk_processor�recreate_rechunkifyFr;rrg�������?)rsrv)rQ)r��getattrr;r�r{rTr�rdrmr�r�rF�chunk_increfr\rQ)rMrsr�rTZ
chunk_iterr�r�Zpart_numberrwZdummy_statsr�rN)r�rMrTrO�
chunk_file�s*


zArchive.chunk_filecCs�t|�}|dkrtd|��t|�}|dkr8td|��ttj��d}t||d@dB|||||||d�	}	tjj}
|j	|	||j
t|jj
|
���|	jdd�|j
jd	7_|j|	�d
S)Nzno such user: %szno such group: %siʚ;i��i�)	rcr�rfrdrgrermr_r�T)r�rr)r rr"r�rdr9rq�stdinr�r�rTr�r�r�r�rKr)rMrcr�r�rdrerfrg�trsrFrNrNrO�
process_stdins&

zArchive.process_stdincs��j||d����\}}}}|j�j|��t|j�}|rLtjtj|j�B|_|sX|�r�|s�t	t
jj�j
|��}	�jj|	�}
�j|	|
|�\}}nd}	}
d
\}}d}
|dk	r�xB|D]}�j|�s�d}Pq�W��fdd�|D�}
d}n|r�dnd}||_|
dk	�r|
|_nxtd��tj|�}WdQRXt
j|d��&}�j|��jt�jj||���WdQRX|�s��j|	|
|d	d�|jD���jjd
7_|j�j||��|j dd�|SQRXdS)NF�Mcsg|]}�j|�j��qSrN)r�rT)r�r�)r�rMrNrOr�Gsz(Archive.process_file.<locals>.<listcomp>�U�ArD�rbcSsg|]
}|j�qSrN)r\)r�r�rNrNrOr�WsrT)r�)FN)!r�rSr�r�rKr�S_IFREG�S_IMODEr�r*r�rcr2r�r�r�Zfile_known_and_unchangedZ
seen_chunkr�r�r�r��_open_rb�fdopenr�rTr�r�r�Z
memorize_filerKr�r�)rMrcr[r�rsr�r�r�Zis_special_fileZhashed_pathZ	path_hashZknownr�r�r�ZfhrFrN)r�rMrO�process_file+sD




*zArchive.process_fileccs(x"|jD]}t|||||d�VqWdS)N)r�)r�r�)r�r�r�r�r�rNrNrO�
list_archives]szArchive.list_archivescCs�ytj|t�Stk
r2ttkr&�tj|t�Stk
r|}z0dtkrj|jtjkrjttkrjtj|t�S�WYdd}~XnXdS)NZretry_erofs)	r�rD�
flags_noatime�PermissionError�flags_normalr�r5r�ZEROFS)rc�excrNrNrOr�cszArchive._open_rb)N)NFFN)T)NNNN)TFFFNrNN)FN)FF)NT)N)N)0r[r|r}rr�r�r��CHUNKER_PARAMSrPr	r�r~rrrrrrrWr]rrrr!rrrr:r]rQrwrxr�r�r�r�r�r�r�r�r�r�r�r�r��staticmethodr�r�rNrNrNrOr�8s`

5	 

	

.

^

E

$2r�cs�t|�}|dkrdS|dd@dkr*d}n|ddkr<d}ndS||krLdS||d@d	kr^n||dkrlndS||d
��t�fdd�|D��S)z1check if the data <d> looks like a msgpacked dictrF���r��r-���������Nc3s|]}�j|�VqdS)N)rI)r��pattern)�key_serializedrNrOr��sz'valid_msgpacked_dict.<locals>.<genexpr>)r�r�r�)r��any)r�Zkeys_serializedZd_lenZoffsrN)r�rO�valid_msgpacked_dictvs"r�csPeZdZdZGdd�de�Z�fdd�Zdd�Zdd	�Zd
d�Z	dd
�Z
�ZS)�RobustUnpackerzCA restartable/robust version of the streaming msgpack unpacker
    c@seZdZdZdS)zRobustUnpacker.UnpackerCrashedzraise if unpacker crashedN)r[r|r}r�rNrNrNrO�UnpackerCrashed�sr�cs>t�j�dd�|D�|_||_g|_d|_tjtd�|_	dS)NcSsg|]}tj|j���qSrN)r4�packb�encode)r�r�rNrNrOr��sz+RobustUnpacker.__init__.<locals>.<listcomp>F)�object_hook)
r�rP�	item_keys�	validator�_buffered_data�_resyncr4r�r.�	_unpacker)rMr�r�)r�rNrOrP�s
zRobustUnpacker.__init__cCsg|_d|_dS)NT)r�r�)rMrNrNrO�resync�szRobustUnpacker.resynccCs$|jr|jj|�n|jj|�dS)N)r�r�r�r�r�)rMrwrNrNrOr��szRobustUnpacker.feedcCs|S)NrN)rMrNrNrO�__iter__�szRobustUnpacker.__iter__cs��fdd�}�jr�dj�j�}x��jr�|s.t�t|�j�sH|dd�}q tjtd��_	�j	j
|�y
|�}Wn�jtfk
r�YnX�j|�r�d�_|S|dd�}q Wn|�SdS)NcsDy
t�j�Sttfk
r>}z�jt|���WYdd}~XnXdS)N)r�r�r�r�r�r�)r')rMrNrO�unpack_next�s
z,RobustUnpacker.__next__.<locals>.unpack_next�r)r�F)
r�r2r�r�r�r�r4r�r.r�r�r�r�)rMr�rwrsrN)rMrO�__next__�s(

zRobustUnpacker.__next__)r[r|r}r�rr�rPr�r�r�r�r�rNrN)r�rOr��sr�c@sZeZdZdd�Zddd�Zd	d
�Zdd�Zd
d�Zdd�Zddd�Z	dd�Z
ddd�ZdS)�ArchiveCheckercCsd|_t�|_dS)NF)�error_foundr��possibly_superseded)rMrNrNrOrP�szArchiveChecker.__init__FNrrbc
CsNtjd�|dko t|||f�|_||_||_|j�|jsLtjd�dS|j	|�|_
|rd|j�tj
|jkr�tjd�d|_|j�|_nly"tj|tjjf|j
d�\|_}
WnHtk
r�}z,tjd|�d|_|jtj
=|j�|_WYdd}~XnX|j|||||d	�|j�|j|	d
�|j�r4tjd�n
tjd�|j�pL|jS)
a�Perform a set of checks on 'repository'

        :param repair: enable repair mode, write updated or corrected data into repository
        :param archive: only check this archive
        :param first/last/sort_by: only check this number of first/last archives ordered by sort_by
        :param glob: only check archives matching this glob
        :param verify_data: integrity verification of data referenced by archives
        :param save_space: Repository.commit(save_space)
        z%Starting archive consistency check...NzJRepository contains no apparent data at all, cannot continue check/repair.FzRepository manifest not found!T)r�z$Repository manifest is corrupted: %s)r�first�last�sort_by�glob)�
save_spacez3Archive consistency check complete, problems found.z6Archive consistency check complete, no problems found.)rprr��	check_all�repairr��init_chunksr�r{�make_keyr��verify_datar�MANIFEST_IDr��rebuild_manifestr�r�Z	OperationZCHECKr�rebuild_refcounts�orphan_chunks_checkr/)rMr�r�rr�r�r�r�r�r�r�r�rNrNrO�check�s:


"

zArchiveChecker.checkcCsttt|j�tjd�}t|�|_d}xH|jjt|d�}|s>P|d}tdddd�}x|D]}||j|<qZWq(WdS)z8Fetch a list of all object keys from repository
        g�������?N)�limit�markerrr)�refcountrQrIr�)	r�r�r�rZMAX_LOAD_FACTORr�r�rCr)rMZcapacityr��resultZ
init_entryr�rNrNrOr�s

zArchiveChecker.init_chunkscCszd}xR|jj�D]D\}}|d7}|dkr*P|j|�}y
t||�Stk
rRYqXqW|dkrfd}nd|}t|��dS)Nrri�z*make_key: repository has no chunks at all!z4make_key: failed to create the key (tried %d chunks))r��	iteritemsr�rrr)rMr�ZattemptZchunkidr��cdatarxrNrNrOr�s


zArchiveChecker.make_keycCs�tjd�t|j�}d}d}g}t|dddd�}d}�xf|jjd|d�}|sNP|t|�7}|d}|jj|�}tt	|��}	�x|	�r�|j
�|	jd�}
yt|�}Wn|t
jtfk
�r}zXd
|_|d	7}tjdt|
�|�t|t�r�|j|
�|	�rtt	|	��}|jj|�}WYdd}~Xq~X|
tjk�r0dn|
}
y|jj|
|�Wq~tk
�r�}z0d
|_|d	7}tjdt|
�|�|j|
�WYdd}~Xq~Xq~Wq8W|j�||k�r�tjd
�tjd||�|�r�|j�rptjd�x�|D]�}y2|jj|�}|tjk�rdn|}
|jj|
|�Wn:tk
�rV|j|=|jj|�tjdt|��YnXtjdt|���q�Wn*tjd�x|D]}tjdt|���q�W|�r�tjntj}|d||�dS)Nz5Starting cryptographic data integrity verification...rzVerifying data %6.2f%%g{�G�z�?zcheck.verify_data)r*rx�stepr+�d)r�r�rTzchunk %s: %szchunk %s, integrity error: %szGRepo/Chunks index object count vs. segment files object count mismatch.z:Repo/Chunks index: %d objects != segment files: %d objectszmFound defect chunks. They will be deleted now, so affected files can get repaired now and maybe healed later.zchunk %s deleted.z0chunk %s not deleted, did not consistently fail.z}Found defect chunks. With --repair, they would get deleted, so affected files could get repaired then and maybe healed later.zchunk %s is defect.z`Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.r�r�)rprr�r�r2r��scanr�r��reversedr.�popr�rBrzrr�r{r/�
isinstancer�rr�r�r�r/r�rqr�r��debug)rMZchunks_count_indexZchunks_count_segments�errorsZ
defect_chunksr1r�Z	chunk_idsZchunk_data_iterZchunk_ids_revdr�Zencrypted_datar'Z	_chunk_id�integrity_errorZdefect_chunk�logrNrNrOr�%s|





$





zArchiveChecker.verify_datac)s*tdd�tD����fdd�}tjd�t|j|j�}dd�tD�}tt	|j
�dd	d
d�}�x�|j
j�D�]�\}}|j�|jj
|�}y|jj||�}Wn6tk
r�}	ztjd|	�d
|_whWYdd}	~	XnXt||�s�qhd|kshd|kr�qhytj|�}
Wntttfk
�rwhYnX||
�rhy|jj|dd�\}
}}WnXtk
�r�}z:|
j
dd�jdd�}
tjd|
|�tjd�d
|_whWYdd}~XnXt|
d�}
|
j}
tjd|
�|
|jk�rd}x(d|
|f}||jk�r�P|d7}�q�Wtjd|
|�|}
||
jf|j|
<qhW|j�tjd�|S)z�Rebuild the manifest object if it is missing

        Iterates through all objects in the repository looking for archive metadata blocks.
        css|]}|j�VqdS)N)r�)r�r�rNrNrOr�{sz2ArchiveChecker.rebuild_manifest.<locals>.<genexpr>cs t|t�sdSt|�}�j|�S)NF)r�r�r��issubset)�obj�keys)�required_archive_keysrNrO�
valid_archive}s
z6ArchiveChecker.rebuild_manifest.<locals>.valid_archivez9Rebuilding missing manifest, this might take some time...cSsg|]}tj|j���qSrN)r4r�r�)r�r�rNrNrOr��sz3ArchiveChecker.rebuild_manifest.<locals>.<listcomp>zRebuilding manifest %6.2f%%g{�G�z�?zcheck.rebuild_manifest)r*rxr�r+zSkipping corrupted chunk: %sTNscmdlines	�versionF)rsnames	<unknown>�asciirz3Archive TAM authentication issue for archive %s: %szMThis archive will *not* be added to the rebuilt manifest! It will be deleted.)r�zFound archive %srz%s.%dz(Duplicate archive name %s, storing as %szManifest rebuild complete.) �	frozensetZREQUIRED_ARCHIVE_KEYSrprrr�r�ZARCHIVE_KEYSr2r�r�r�r.r�r�rr{r�r�r4Zunpackbr�r�r�rrrr:r�r�rqrdr/)rMrr�Zarchive_keys_serializedr1r�r�r�rwr�r�verifiedr�r�r�new_namerN)rrOr�vsb




zArchiveChecker.rebuild_manifestc1s��jjtjd��fdd����fdd�}d(�fdd�	����fdd	�}��fd
d�}|dkr�|jd�}t|||f�r�jjj||||d
�}	|r�|	r�t	j
d|�|r�t|	�|kr�t	j
d|t|	��|r�t|	�|kr�t	j
d|t|	��n�jjj|d�}	n<y�jj|g}	Wn(tk
�r4t	j
d|�d�_dSXt|	�}
t|
dddd�}t�j���d��xRt|	�D�]D\}}
|j|�t	jdj|
j|d|
��|
j}|�jk�r�t	j
dt|��d�_�jj|
j=�qh�|��jj|�}y�jj||�}WnLtk
�r@}z.t	j
dt|�|�d�_�jj|
j=�whWYdd}~XnXy�jj|dd�\}}}WnTtk
�r�}z6t	j
d|
j|�t	j
d�d�_�jj|
j=�whWYdd}~XnXt|d �}|jdk�r�t d!��d"d#�|j!D�|_!t"�j�}||_#x2||�D]&}d$|k�r||
j|�|j$|��q�W|j%dd%�x|j&D]}�|��q:W|j|_&�jj'|j(�d&|d'�}�jj)|�}�jj*|�}�|t|�t|�|�||
j+f�jj|
j<�qhW|j,�WdQRXdS))z�Rebuild object reference counts by walking the metadata

        Missing and/or incorrect data is repaired when detected
        Ncs,�jj|tddd��jdkr(�jj|�dS)Nr)r�r�rr�r�r�)r�)rMrNrO�mark_as_possibly_superseded�szEArchiveChecker.rebuild_refcounts.<locals>.mark_as_possibly_supersededcs2�jj|�}�jj|�}�|t|�t|�|�|S)N)r�r��encryptr�)r�r�r�)�
add_referencerMrNrO�add_callback�sz6ArchiveChecker.rebuild_refcounts.<locals>.add_callbackcs^y�jj|�WnHtk
rX|dk	s,t�td||d��j|<�jrT�jj||�YnXdS)Nr)r�rQrI)r�Zincrefrr�rr�r�Zput)r�rQrIr�)rMrNrOr�sz7ArchiveChecker.rebuild_refcounts.<locals>.add_referencec	sr�fdd�}d}g}d}d|k}|j}|r0|jn|}|rht|�t|�krhtjdj||j��|`d}|}�x~t||�D�]n\}	}
|
\}}}
|�jk�r||	|
kr�tjdj||j|||t	|���d�_
}||�\}}}
}�|||
|�n�tjd	j||j|||t	|���|	\}}}
|�jk�r0�|||
�nJtjd
j||j|||t	|���d�_
}||�\}}}
}�|||
|�nR|	|
k�r��|||
�n:tjdj||j|||t	|����|||
��|	d�|j|||
g�||7}qvW|�r|�r|j|_|�r(||k�r(tjdj||j��|`||_d
|k�rn|j
}|jddd�}||k�rntjdj||j||��dS)aVerifies that all file chunks are present.

            Missing file chunks will be replaced with new chunks of the same length containing all zeros.
            If a previously missing file chunk re-appears, the replacement chunk is replaced by the correct one.
            cs4t|�}�jj|�}�jj|�}t|�}||||fS)N)r�r�r�r
r�)rQrwr�r�rI)rMrNrO�replacement_chunk�s
zWArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunks.<locals>.replacement_chunkrFr;z0{}: {}: Invalid chunks_healthy metadata removed!z^{}: {}: New missing file chunk detected (Byte {}-{}, Chunk {}). Replacing with all-zero chunk.Tz|{}: {}: Previously missing file chunk is still missing (Byte {}-{}, Chunk {}). It has an all-zero replacement chunk already.zm{}: {}: Missing all-zero replacement chunk detected (Byte {}-{}, Chunk {}). Generating new replacement chunk.zE{}: {}: Healed previously missing file chunk! (Byte {}-{}, Chunk {}).z2{}: {}: Completely healed previously damaged file!rQ)Z
compressedr�z<{}: {}: size inconsistency detected: size {}, chunks size {}N)r�r;r�rprqrVrcr�r{r/r�rr�rQr�)�archive_namersr
�offsetZ
chunk_listZchunks_replacedZhas_chunks_healthyZchunks_currentr;Z
chunk_currentZ
chunk_healthyr�rQrIr�rZrY)rr	rMrNrO�verify_file_chunks�sf






z<ArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunksc3s�tdd��jjD���tdd�tD���tdd��jj�}d���fdd�}�fd	d
�}dd�����fd
d�}d}�xdt|j|�D�]R\}}t|�}|dr�x |D]}|d||�|d7}q�Wq�|dkr�|j��xt	|�j
|��D]�\}}	y\�jj||	�}
|j
|
�x>|D]6}||�\}}
|�r2t|d�Vn|d|
||��qWWn�tk
�r~}z|t|�||�WYdd}~XnNtjk
�r�|d||�|j�Yn$tk
�r�|d||��YnX|d7}q�Wq�WdS)z�Iterates through all archive items

            Missing item chunks will be skipped and the msgpack stream will be restarted
            css|]}|j�VqdS)N)r�)r�r�rNrNrOr�/szLArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<genexpr>css|]}|j�VqdS)N)r�)r�r�rNrNrOr�0scSst|t�od|kS)Nspath)r�r.)rsrNrNrOr1szKArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<lambda>rcs"�dt|�jk�kr�d7��S)N�r)r�r�)r�)�_staterMrNrO�missing_chunk_detector5szYArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.missing_chunk_detectorcs,t|�}|d||f7}d�_tj|�dS)Nz [chunk: %06d_%s]T)r/r�rpr{)rxr�Zchunk_nor�)rMrNrO�report;szIArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.reportcSsdjdd�|D��S)Nz, css,|]$}t|t�r|jdd�nt|�VqdS)r)r�N)r�r�rrr�)r�rsrNrNrOr�BszdArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safe.<locals>.<genexpr>)r2)rrNrNrO�list_keys_safeAszQArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safecsbt|t�sdS|jdd�t|�}�j|�s@dd��|�fS|j��s^dd�|��fSd	S)
NF�not a dictionarysaclzmissing required keys: zinvalid keys: Trb)Fr)Trb)r�r.r�r�r)rr)r�r�required_item_keysrNrO�
valid_itemDs


zMArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.valid_itemrzitem metadata chunk missingr)r�zDDid not get expected metadata dict when unpacking item metadata (%s)NzCUnpacker crashed while unpacking item metadata, trying to resync...z5Exception while decrypting or unpacking item metadata)rr�r�ZREQUIRED_ITEM_KEYSr�r	r�r�r�r�r�r�r�r�r9rr�r�r)rr�rrrr�stater�r�r�rwrsZvalid�reasonr�)r�rM)rr�rrrO�robust_iterator*sJ




 z9ArchiveChecker.rebuild_refcounts.<locals>.robust_iterator�,)r�r�r�r�z.--glob-archives %s does not match any archivesz+--first %d archives: only found %d archivesz*--last %d archives: only found %d archives)r�zArchive '%s' not found.TzChecking archives %3.1f%%g�������?zcheck.rebuild_refcounts)r*rxr�r+zAnalyzing archive {} ({}/{})rz%Archive metadata block %s is missing!z*Archive metadata block %s is corrupted: %sF)rz3Archive TAM authentication issue for archive %s: %szEThis archive will be *removed* from the manifest! It will be deleted.)r�z Unknown archive metadata versioncSsg|]}t|��qSrN)r+)r�r
rNrNrOr��sz4ArchiveChecker.rebuild_refcounts.<locals>.<listcomp>r�)rlsarchive)r"�salt)N)-r�r�rr�r3r�r�r�r�rprqr�rr{r�r2rAr�r�r.rrVr�r\r/r�r�r�rrr:rrrr�r�r�rlr�r#r^r�r
rr/)rMrr�r�r�r�rrrZ
archive_infosZnum_archivesr1rrZ
archive_idr�rwr�rrr�rsZprevious_item_idZnew_archive_idrN)rr	r�rMrOr��s�	LH






z ArchiveChecker.rebuild_refcountscCs�|jr�dd�|jj�D�}||j}|rBtjdjt|���d|_|j	r�|r�tj
dt|�t|j�f�x|D]}|jj|�qnWtj
d�n
tj
d�dS)NcSsh|]\}}|jdkr|�qS)r)r�)r�r�r(rNrNrO�	<setcomp>�sz5ArchiveChecker.orphan_chunks_check.<locals>.<setcomp>z{} orphaned objects found!Tz1Deleting %d orphaned and %d superseded objects...z.Finished deleting orphaned/superseded objects.z<Orphaned objects check skipped (needs all archives checked).)
r�r�r�r�rpr{rVr�r�r�rr�r�)rMZunusedZorphanedr�rNrNrOr��s


z"ArchiveChecker.orphan_chunks_checkcCs@|jr<tjd�|jj�tjd�|jj|d�tjd�dS)NzWriting Manifest.z>Committing repo (may take a while, due to compact_segments)...)r�zFinished committing repo.)r�rprr�r�r�r$)rMr�rNrNrOr/�s


zArchiveChecker.finish)FNrrrbNFF)NrrrbN)F)r[r|r}rPr�r�r�r�r�r�r�r/rNrNrNrOr��s
*QM
{r�c
@s�eZdZGdd�de�Zedd��Zd!dd	�Zd"d
d�Zdd
�Z	dd�Z
dd�Zdd�Zdd�Z
d#dd�Zdd�Zd$dd�Zdd�Zdd �ZdS)%�ArchiveRecreaterc@seZdZddd�ZdS)zArchiveRecreater.InterruptedNcCs|pi|_dS)N)r)rMrrNrNrOrP�sz%ArchiveRecreater.Interrupted.__init__)N)r[r|r}rPrNrNrNrO�Interrupted�sr cCs
|jd�S)Nz	.recreate)�endswith)rrNrNrO�is_temporary_archive�sz%ArchiveRecreater.is_temporary_archiveFN�cCs�||_||_||_||_||_||_|p*g|_||_|	dk	|_|jrPt	j
d|	�|	pVt|_||_
||_|
pptd�|_t�|_||_|
|_||_||_|p�dd�|_|
r�dn||_dS)NzRechunking archives to %sZnonecWsdS)NrN)�argsrNrNrOr�sz+ArchiveRecreater.__init__.<locals>.<lambda>)r�r�r�r��matcher�exclude_caches�exclude_if_present�keep_exclude_tags�
rechunkifyrpr�r�r��
recompress�always_recompressr�compressionr��seen_chunksr%rXrTr�print_file_statusr�)rMr�r�r�r�r%r&r'r(r�r,r*r+rXrTrZfile_status_printerr%r�rNrNrOrP�s,


zArchiveRecreater.__init__cCs�|j|�st�|j|�}|j||�}|js2|jr<|j|�|jj�rb|j	rb|j
rb|dkrbdS|j||�|dk}|j||||d�dS)NF)�replace_originalT)
r"r��open_archive�
create_targetr'r&�matcher_add_tagged_dirsr%�emptyr*r��
process_itemsr)rMrr��target_namer�targetr/rNrNrO�recreate�s

"zArchiveRecreater.recreatec
s|j}|j���rind}�fdd�}x�|j�D]�}|j|j�sv|jd|j�||�r2|jd�|jd�df||j<q2�r�t|j�r�|jd�|kr�||j	\}}}	|	dkr�||_
|dk	r�||_dd|jf||j	<|`	n|	|_	|jr�|jd|j�q2|j
|||�q2W|j�r|jjdd	�dS)
Ncs"�o t|j�o |jdd�o d|kS)Nr�Tr�)rr�r�)rs)�target_is_subsetrNrO�item_is_hardlink_masters
z?ArchiveRecreater.process_items.<locals>.item_is_hardlink_master�xr�r;r��-T)rt)r%r3r�matchrcr.r�rr�r�r�r;rX�process_itemrrTr{)
rMrr6r%r�r9rsr�r;Z
new_sourcerN)r8rOr4s0
zArchiveRecreater.process_itemscCsHd|kr&|j|||�|jjd7_|j|�|jt|j�|j�dS)Nr�r)�process_chunksrTrKrr.r(r�rc)rMrr6rsrNrNrOr=0s

zArchiveRecreater.process_itemc	Csv|jr<|jr<x$|jD]\}}}|jj||j�qW|jS|j||t|j��}t|j	|�}|j
||j|j||�dS)N)r*r�r�r�r�rT�iter_chunksr�rr�r�)	rMrr6rsr�rQrI�chunk_iteratorr�rNrNrOr>7szArchiveRecreater.process_chunkscCs�|jj|�}||jkr&|jj||j�S|j}|jr�|jr�||jjkr�|jj	d|j
j|�dd�}tj
|�j|jjj|�jkr�d}|jj|||j|dd�}|jj
jdd�|jj|j�|S)NF)Z
decompress)�	overwriter�)r�)r�r�r-r�r�rTr*r+r�r�r�r�rZdetectr�Z
compressorZdecider�r�r�r\)rMr6rwr�rAZ	old_chunkr�rNrNrOr�@s
z ArchiveRecreater.chunk_processorccsP|jjdd�|D��}|jr8t|�}|jj|�EdHnx|D]
}|Vq>WdS)NcSsg|]\}}}|�qSrNrN)r�r�r�rNrNrOr�Qsz0ArchiveRecreater.iter_chunks.<locals>.<listcomp>)r�r�r�rr�r�)rMrr6r�r@rkr�rNrNrOr?Ps
zArchiveRecreater.iter_chunksTcCs�|jr
dS|dkr |jjdd�}|jr,|j}|jdkr`|jj|jjd�pN|jj|jjtj	d�}n|jjtj	d�}|j
||j|d�|r�|jt�|j
d�|j|j�|jr�||_tj�|_ttt|�tt|j�t|j�t�dS)Nr�rbr
)rdr
r�recreate_cmdline)rrB)r�r%r&)r)rXrr�rTr�r%rdrrqrrr�rFrrxr�rr�rjr3ZDASHESr�r�)rMrr6r�r/Z_startr&rNrNrOr[s6



zArchiveRecreater.savec	s>���fdd�}�j�g�g�i}�jrhx>|jdd�d�D]*}tj|j�r:d|kr:d|kr:d||j<q:Wx�|j�fd	d�d�D]�}�jr�|j|kr�|||j<tjj	|j�\}}|�j
kr�|||�q~�jo�|tko�tj|j�r~d|kr�|n||j}t||�}|j
tt��tkr~|||�q~W�j�tj��j�tj�dS)
zLAdd excludes to the matcher created by exclude_cache and exclude_if_present.csH�jr2�jt|jdd���jt|ddd��n�jt|dd��dS)NF)Zrecurse_dirr<)r(r�r6rcr7)�dirZtag_item)rM�	tag_files�tagged_dirsrNrO�exclude�sz9ArchiveRecreater.matcher_add_tagged_dirs.<locals>.excludecSstjj|j�tkS)N)r�rc�basename�CACHE_TAG_NAME)rsrNrNrOr�sz:ArchiveRecreater.matcher_add_tagged_dirs.<locals>.<lambda>)r�r�r�Ncstjj|j�tkp�j|j�S)N)r�rcrGrHr<)rs)r%rNrOr�s)r%r&rrrPr�r�rcr�r3r'rHrr�r�ZCACHE_TAG_CONTENTSr�r8ZIncludeZExcludeNoRecurse)	rMrrFZcachedir_mastersrsrCZtag_fileZcontent_itemrkrN)r%rMrDrErOr2�s.


z(ArchiveRecreater.matcher_add_tagged_dirscCsZ|p|jd}|j|�}t|jjdg��}|jo8||jk|_|jrVtj	d|pNd|j�|S)zCreate target archive.z	.recreater�z Rechunking archive from %s to %sz	(unknown))
r��create_target_archive�tuplerr�r)r�r�rpr�)rMrr5r6Zsource_chunker_paramsrNrNrOr1�s
zArchiveRecreater.create_targetcCs,t|j|j|j|d|j|j|j|jd�	}|S)NT)r�rr�r�r�)r�r�r�r�rr�r�r�)rMr�r6rNrNrOrI�s
z&ArchiveRecreater.create_target_archivecKs"t|j|j|j|fd|ji|��S)Nr�)r�r�r�r�r�)rMr��kwargsrNrNrOr0�szArchiveRecreater.open_archive)
FNFNNFFFFFNNr#)NN)NT)N)r[r|r}rr r�r"rPr7r4r=r>r�r?rr2r1rIr0rNrNrNrOr�s"


%	
(*
r)qr�rnr�Zsocketrrqrd�
contextlibrrrr�	functoolsrZgetpassr�ior�	itertoolsr	Zshutilr
rprrbr
r�rr�rZ
crypto.keyrr�compressrrZ	constantsZ	hashindexrrrZhelpersrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5Zpatternsr6r7r8rsr9r:�platformr;r<r=r>r?r@ZremoterAr�rBrCr�r5�O_RDONLYr�r�r�rFr�rr�r�r�r�r�r�r�r�r�r�r�r�rrNrNrNrO�<module>s� 
LD&
D=