| Current File : //usr/lib64/python3.6/site-packages/borg/__pycache__/cache.cpython-36.pyc |
3
up�dC� � @ s* d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm Z ddl
m Z e � Z
e d�Z ddl
mZmZ dd lmZmZmZ dd
lmZ dd lmZ dd lmZ dd
lmZmZ ddlmZmZm Z m Z ddlm Z ddlm Z ddlm Z m!Z! ddlm"Z" ddlm#Z#m$Z$ ddlm%Z%m&Z& ddlm'Z' ddlm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 dd l6m7Z7 dd l8m9Z9 ed d �Z:G d d!� d!�Z;d"d#� Z<d$d%� Z=d6d&d'�Z>d(d)� Z?d*d+� Z@G d,d-� d-�ZAG d.d/� d/�ZBG d0d1� d1�ZCG d2d3� d3eC�ZDG d4d5� d5eC�ZEdS )7� N)� unhexlify)�
namedtuple)� perf_counter� )�
create_loggerzborg.debug.files_cache)� CACHE_README�FILES_CACHE_MODE_DISABLED)�
ChunkIndex�ChunkIndexEntry�CacheSynchronizer)�Location)�Error)�Manifest)�
get_cache_dir�get_security_dir)�
int_to_bigint�
bigint_to_int�
bin_to_hex�parse_stringified_list)�format_file_size)�safe_ns)�yes�hostname_is_unique)�remove_surrogates)�ProgressIndicatorPercent�ProgressIndicatorMessage)�set_ec� EXIT_WARNING)� safe_unlink)�msgpack)� ArchiveItem�ChunkListEntry)� PlaintextKey)�IntegrityCheckedFile� DetachedIntegrityCheckedFile�FileIntegrityError)�Lock)�SaveFile)�cache_if_remote)�LIST_SCAN_LIMIT�FileCacheEntryz age inode size cmtime chunk_idsc @ s� e Zd ZdZdd� Zeddd��Zdd� Zd d
� Zd d � Z d d
d�Z
d dd�Z d dd�Z dddd�dd�Z
d dd�Zdd� ZdS ) �SecurityManageras
Tracks repositories. Ensures that nothing bad happens (repository swaps,
replay attacks, unknown repositories etc.).
This is complicated by the Cache being initially used for this, while
only some commands actually use the Cache, which meant that other commands
did not perform these checks.
Further complications were created by the Cache being a cache, so it
could be legitimately deleted, which is annoying because Borg didn't
recognize repositories after that.
Therefore a second location, the security database (see get_security_dir),
was introduced which stores this information. However, this means that
the code has to deal with a cache existing but no security DB entry,
or inconsistencies between the security DB and the cache which have to
be reconciled, and also with no cache existing but a security DB entry.
c C sV || _ t|j�| _t|�| _tjj| jd�| _tjj| jd�| _ tjj| jd�| _
d S )Nzkey-type�locationzmanifest-timestamp) �
repositoryr �id_str�dir� cache_dir�os�path�join�
key_type_file�
location_file�manifest_ts_file)�selfr- � r8 � /usr/lib64/python3.6/cache.py�__init__> s
zSecurityManager.__init__Nc C s( |p t | j�}tjj|�r$tj|� dS )z:destroy the security dir for ``repository`` or at ``path``N)r r. r1 r2 �exists�shutil�rmtree)r- r2 r8 r8 r9 �destroyF s zSecurityManager.destroyc C s t dd� | j| j| jfD ��S )Nc s s | ]}t jj|�V qd S )N)r1 r2 r; )�.0�fr8 r8 r9 � <genexpr>N s z(SecurityManager.known.<locals>.<genexpr>)�allr4 r5 r6 )r7 r8 r8 r9 �knownM s zSecurityManager.knownc C sn | j � s dS y.t| jd��}|j� }|t|j�kS Q R X W n. tk
rh } ztjd|� W Y d d }~X nX d S )NF�rz&Could not read/parse key type file: %s) rC �openr4 �read�str�TYPE�OSError�logger�warning)r7 �key�fd�type�excr8 r8 r9 � key_matchesQ s zSecurityManager.key_matchesc C s� t jd| jj| j� | jjj� }t jd|� t jdt|j�� t jd|j � t
| j ��}|j |� W d Q R X t
| j
��}|j t|j�� W d Q R X t
| j��}|j |j � W d Q R X d S )Nz#security: saving state for %s to %sz security: current location %sz security: key type %sz security: manifest timestamp %s)rJ �debugr- r. r/ � _location�canonical_pathrG rH � timestampr' r5 �writer4 r6 )r7 �manifestrL Zcurrent_locationrM r8 r8 r9 �save[ s zSecurityManager.savec C s, y.t | j��}|j� }W d Q R X tjd|� W nV tk
rT tjd| j� d }Y n2 tk
r� } ztjd|� d }W Y d d }~X nX |r�|jr�||jkr�|j}tjd|� | j j
j � }|o�||k�r(dj ||�d }t
|ddd d
d �s�tj� �tjd � t| j��}|j|� W d Q R X |�r(|j� d S )
Nz#security: read previous location %rz-security: previous location file %s not foundz)Could not read previous location file: %sz.security: using previous_location of cache: %rzDWarning: The repository at location {} was previously located at {}
z Do you want to continue? [yN] z Aborting.zInvalid answer, aborting.FZ BORG_RELOCATED_REPO_ACCESS_IS_OK)� false_msg� invalid_msg�retry�env_var_overridez<security: updating location stored in cache and security dir)rE r5 rF rJ rQ �FileNotFoundErrorrI rK �previous_locationr- rR rS �formatr �Cache�RepositoryAccessAbortedr' rU rW )r7 � cache_configrM r] rO Zrepository_location�msgr8 r8 r9 �assert_location_matchesh s4
z'SecurityManager.assert_location_matchesc C s� y.t | j��}|j� }W d Q R X tjd|� W nV tk
rT tjd| j� d}Y n2 tk
r� } ztjd|� d}W Y d d }~X nX |r�t||j p�d�}tjd|� |r�||j kr�t
|t �r�t j
� �nt j� �d S )Nz$security: read manifest timestamp %rz.security: manifest timestamp file %s not found� z)Could not read previous location file: %sz4security: determined newest manifest timestamp as %s)rE r6 rF rJ rQ r\ rI rK �maxrT �
isinstancer" r_ �RepositoryIDNotUnique�RepositoryReplay)r7 rV rL ra rM rT rO r8 r8 r9 �assert_no_manifest_replay� s"
z)SecurityManager.assert_no_manifest_replayc C sF |r&|j d k r&|j t|j�kr&tj� �| j� rB| j|� rBtj� �d S )N)�key_typerG rH r_ �EncryptionMethodMismatchrC rP )r7 rL ra r8 r8 r9 �assert_key_type� s z SecurityManager.assert_key_typeT)ra �warn_if_unencrypted� lock_waitc C sr | j |||� |r"| j|||� nBt| j|d�}|j� rX|� | j|||� W d Q R X n | j||� tjd� d S )N)rn z/security: repository checks ok, allowing access)�assert_access_unknown�_assert_secure� CacheConfigr- r; rJ rQ )r7 rV rL ra rm rn r8 r8 r9 �
assert_secure� s z SecurityManager.assert_securec C sF | j |� | j||� | j|||� | j� sBtjd� | j||� d S )Nz3security: remembering previously unknown repository)rc rl ri rC rJ rQ rW )r7 rV rL ra r8 r8 r9 rp � s
z SecurityManager._assert_securec C sf |j rb| j� rbd
}| p,t|ddddd�}|rZ|rBtjd� n
tjd � | j||� ntj� �d S ) NzKWarning: Attempting to access a previously unknown unencrypted repository!
z Do you want to continue? [yN] z Aborting.zInvalid answer, aborting.FZ*BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK)rX rY rZ r[ zIsecurity: remembering unknown unencrypted repository (explicitly allowed)z-security: initializing unencrypted repositoryziWarning: Attempting to access a previously unknown unencrypted repository!
Do you want to continue? [yN] )Zlogically_encryptedrC r rJ rQ rW r_ �CacheInitAbortedError)r7 rm rV rL rb Z allow_accessr8 r8 r9 ro � s
z%SecurityManager.assert_access_unknown)N)N)N)N)N)�__name__�
__module__� __qualname__�__doc__r: � staticmethodr> rC rP rW rc ri rl rr rp ro r8 r8 r8 r9 r+ * s
r+ c C s t | �}|j||j|d� d S )N)rn )r+ rr rL )r- rV rn Zsmr8 r8 r9 rr � s rr c C s� |j j� }t|�}t| �}|j|jkr�|j|jkr�|j|jkr�|j|jkr�|jr�|jr�|jjd�r�|jjd�r�|jdd � |jdd � kr�|S | S d S )Nz/~/z/./� ) rR rS r �proto�user�hostZportr2 �
startswith)Zcache_locationr- Z
repo_locationZrlZclr8 r8 r9 � recanonicalize_relative_location� s
0 4r~ c C s |pt jjt� | j�S )N)r1 r2 r3 r r. )r- r2 r8 r8 r9 r0 � s r0 c C s t jjdd�} | rd| S dS )NZBORG_FILES_CACHE_SUFFIXrd zfiles.�files)r1 �environ�get)�suffixr8 r8 r9 �files_cache_name� s r� c C s dd� t j| �D �d S )Nc S s" g | ]}|d ks|j d�r|�qS )r zfiles.)r} )r? �fnr8 r8 r9 �
<listcomp>� s z-discover_files_cache_name.<locals>.<listcomp>r )r1 �listdir)r2 r8 r8 r9 �discover_files_cache_name� s r� c @ s` e Zd Zddd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z ddd�Z
dd� Z dd� Z dS )rq Nc C s4 || _ t||�| _tjj| jd�| _d | _|| _d S )N�config)r- r0 r2 r1 r3 � config_path�lockrn )r7 r- r2 rn r8 r8 r9 r: � s
zCacheConfig.__init__c C s | j � | S )N)rE )r7 r8 r8 r9 � __enter__� s zCacheConfig.__enter__c C s | j � d S )N)�close)r7 �exc_type�exc_val�exc_tbr8 r8 r9 �__exit__� s zCacheConfig.__exit__c C s t jj| j�S )N)r1 r2 r; r� )r7 r8 r8 r9 r; � s zCacheConfig.existsc C s� | j � st�tjd d�}|jd� |jddd� |jdd| jj� |jddd� |jd� |jddd� t| j ��}|j
|� W d Q R X d S ) N)�
interpolation�cache�version�1r- rV rd � integrity) r; �AssertionError� configparser� ConfigParser� add_section�setr- r. r' r� rU )r7 r� rM r8 r8 r9 �create s
zCacheConfig.createc C s2 t tjj| jd�d| jt� d�j� | _| j� d S )Nr� T)� exclusiveZtimeoutZkill_stale_locks) r& r1 r2 r3 rn r �acquirer� �load)r7 r8 r8 r9 rE s zCacheConfig.openc C st t jd d�| _t| j��}| jj|� W d Q R X | j| j� | jjdd�| _t | jjdd��| _
| jjddd d�| _ | jjddd d�| _ t
t| jjddd d���| _t
t| jjdd
d d���| _y@t| jjd ��| _| jjdd�| jjd�kr�i | _tjd � W n( t jk
�r( tjd
� i | _Y nX | jjddd d�}|�rRt|| j�| _nd | _| jj
dd| jj j � � d S )N)r� r� r- rV rT )Zfallbackrj �ignored_featuresrd �mandatory_featuresr� zHCache integrity data not available: old Borg version modified the cache.zTCache integrity: No integrity data found (files, chunks). Cache is from old version.r] ) r� r� �_configrE r� Z read_file�_check_upgrader� �idr � manifest_idrT rj r� r r� r� �dict�itemsr� �poprJ rK �NoSectionErrorrQ r~ r- r] rR rS )r7 rM r] r8 r8 r9 r� s.
zCacheConfig.loadc C s� |r�| j jdd|j� | j jdd|j� | j jdddj| j�� | j jdddj| j�� | j jd�sp| j jd� x&| j j
� D ]\}}| j jd||� q|W | j jdd|j� |r�| j jddt |j �� t
| j��}| j j|� W d Q R X d S ) Nr� rV rT r� �,r� r� rj )r� r� r. rT r3 r� r� Z has_sectionr� r� r� rG rH r' r� rU )r7 rV rL �file�integrity_datarM r8 r8 r9 rW 3 s zCacheConfig.savec C s | j d k r| j j� d | _ d S )N)r� �release)r7 r8 r8 r9 r� C s
zCacheConfig.closec C sj y8| j jdd�}d}||kr6| j� td|||f ��W n, tjk
rd | j� td| �d �Y nX d S )Nr� r� r z0%s has unexpected cache version %d (wanted: %d).z#%s does not look like a Borg cache.)r� Zgetintr� � Exceptionr� r� )r7 r� Z
cache_versionZwanted_versionr8 r8 r9 r� H s zCacheConfig._check_upgrade)NN)NN)
rt ru rv r: r� r� r; r� rE r� rW r� r� r8 r8 r8 r9 rq � s
rq c @ s� e Zd ZdZG dd� de�ZG dd� de�ZG dd� de�ZG dd � d e�ZG d
d � d e�Z e
dd
d��Z e
ddd��Z d ddddd de
df dd�Zd S )r_ zClient Side cache
c @ s e Zd ZdZdS )zCache.RepositoryIDNotUniquez`Cache is newer than repository - do you have multiple, independently updated repos with same ID?N)rt ru rv rw r8 r8 r8 r9 rg X s rg c @ s e Zd ZdZdS )zCache.RepositoryReplayz�Cache, or information obtained from the security directory is newer than repository - this is either an attack or unsafe (multiple repos with same ID)N)rt ru rv rw r8 r8 r8 r9 rh [ s rh c @ s e Zd ZdZdS )zCache.CacheInitAbortedErrorz Cache initialization abortedN)rt ru rv rw r8 r8 r8 r9 rs ^ s rs c @ s e Zd ZdZdS )z Cache.RepositoryAccessAbortedzRepository access abortedN)rt ru rv rw r8 r8 r8 r9 r` a s r` c @ s e Zd ZdZdS )z Cache.EncryptionMethodMismatchzLRepository encryption method changed since last access, refusing to continueN)rt ru rv rw r8 r8 r8 r9 rk d s rk Nc C s( t | |�}ttjj|d�dd�j� d S )Nr� T)r� )r0 r& r1 r2 r3 �
break_lock)r- r2 r8 r8 r9 r� g s
zCache.break_lockc C sH |pt jjt� | j�}t jj|d�}t jj|�rDt j|� tj|� dS )z3destroy the cache for ``repository`` or at ``path``r� N) r1 r2 r3 r r. r; �remover< r= )r- r2 r� r8 r8 r9 r> l s
z
Cache.destroyTFc
s� | rd� krd� n"| r6d� kr6dj t� �td� �� � ��������f dd�}
����fdd�}|
sn|
� S t����}|j� r�|� |j�jk}W d Q R X |r�tjd� |
� S tjd � |� S )
N�d�ird c s t ��������� d� S )N) r- rL rV r2 �syncrm �progressrn �
cache_mode)�
LocalCacher8 ) r� rL rn rV r2 r� r- r� rm r8 r9 �local~ s z Cache.__new__.<locals>.localc s t �� ��d�S )N)r- rL rV rn )�
AdHocCacher8 )rL rn rV r- r8 r9 �adhoc� s z Cache.__new__.<locals>.adhocz%Cache: choosing local cache (in sync)zKCache: choosing ad-hoc cache (local cache does not exist or is not in sync))r3 r� rq r; r� r� rJ rQ )�clsr- rL rV r2 r� Zdo_filesrm r� rn Zpermit_adhoc_cacher� Z ignore_inoder� r� ra Z
cache_in_syncr8 ) r� rL rn rV r2 r� r- r� rm r9 �__new__u s"
z
Cache.__new__)N)N)rt ru rv rw r
rg rh rs r` rk rx r� r> r r� r8 r8 r8 r9 r_ U s r_ c @ sF e Zd ZdZdd� Zedddddd d
g�Zd d � Zd
d� Zdd� Z dS )�CacheStatsMixinz�All archives: {0.total_size:>20s} {0.total_csize:>20s} {0.unique_csize:>20s}
Unique chunks Total chunks
Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}c C s | j j| j� �S )N)�
str_formatr^ � format_tuple)r7 r8 r8 r9 �__str__� s zCacheStatsMixin.__str__�Summary�
total_size� total_csizeZ unique_size� unique_csizeZtotal_unique_chunksZ total_chunksc C s | j | jj� � j� }|S )N)r� �chunksZ summarize�_asdict)r7 �statsr8 r8 r9 r� � s zCacheStatsMixin.statsc C s2 | j � }x dD ]}t|| �||< qW | jf |�S )Nr� r� r� )r� r� r� )r� r r� )r7 r� Zfieldr8 r8 r9 r� � s
z CacheStatsMixin.format_tuplec C s | j � d S )Nr� )r� )r7 r8 r8 r9 �chunks_stored_size� s z"CacheStatsMixin.chunks_stored_sizeN)
rt ru rv r� r� r r� r� r� r� r8 r8 r8 r9 r� � s r� c @ s� e Zd ZdZdddddefdd�Zdd� Zd d
� Zd d � Zd
d� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd/d#d$�Zd0d%d&�Zd1d'd(�Zd2d)d*�Zd+d,� Zd-d.� ZdS )3r� z0
Persistent, local (client-side) cache.
NTFc
C s� || _ || _|| _|| _| | _d| _d| _t||�| _t |�| _
t | j | j|�| _ t
jj| j�sx| j
j|||� | j� | j� yT| j
j||| j d� | j� s�| j� | j� |r�| jj| j jkr�| j� | j� W n | j� � Y nX dS )aO
:param warn_if_unencrypted: print warning if accessing unknown unencrypted repository
:param lock_wait: timeout for lock acquisition (int [s] or None [wait forever])
:param sync: do :meth:`.sync`
:param cache_mode: what shall be compared in the file stat infos vs. cached stat infos comparison
NF)ra )r- rL rV r� r� rT �
txn_activer0 r2 r+ �security_managerrq ra r1 r; ro r� rE rr �check_cache_compatibility�
wipe_cache�update_compatibilityr� r� r� �commitr� )
r7 r- rL rV r2 r� rm r� rn r� r8 r8 r9 r: � s2
zLocalCache.__init__c C s | S )Nr8 )r7 r8 r8 r9 r� � s zLocalCache.__enter__c C s | j � d S )N)r� )r7 r� r� r� r8 r8 r9 r� � s zLocalCache.__exit__c C s� t j| j� tt jj| jd�d��}|jt� W dQ R X | jj� t � jt jj| jd�� t jt jj| jd�� t
t jj| jt � �dd�� W dQ R X dS )z0Create a new empty cache at `self.path`
ZREADME�wNr� zchunks.archive.dT)�binary) r1 �makedirsr2 rE r3 rU r ra r� r r' r� )r7 rM r8 r8 r9 r� � s
zLocalCache.createc C sd | j j� ttjj| jd�d| j jjd�d��}tj |�| _
W d Q R X d| j krXd | _ n| j
� d S )Nr� F)r2 rU r� r� )ra r� r# r1 r2 r3 r� r� r rF r� r� r � _read_files)r7 rM r8 r8 r9 �_do_open� s
zLocalCache._do_openc C s2 t jj| j�s td| j ��| jj� | j� d S )Nz"%s Does not look like a Borg cache)r1 r2 �isdirr� ra rE �rollback)r7 r8 r8 r9 rE s
zLocalCache.openc C s | j d k r| j j� d | _ d S )N)ra r� )r7 r8 r8 r9 r� s
zLocalCache.closec
C s� i | _ d | _tjd� tjd� d }y�ttjj| jt � �d| j
j j t � �d���}t
jdd�}x�|jd�}|snP |j|� y<x6|D ].\}}t|� }t
j|j|jd d
��| j |< q�W W q^ ttfk
r� } zd t|� }P W Y d d }~X q^X q^W W d Q R X W n^ tk
�r* } zd t|� }W Y d d }~X n0 tk
�rX } zd
t| � }W Y d d } ~ X nX |d k �r~tj|� tjd� i | _ tjdt| j �� d S )NzReading files cache ...z FILES-CACHE-LOAD: starting...F)r2 rU r� T)Zuse_list�@ i r )�agez#The files cache seems invalid. [%s]z#The files cache can't be read. [%s]z"The files cache is corrupted. [%s]z:Continuing without files cache - expect lower performance.z.FILES-CACHE-LOAD: finished, %d entries loaded.i ) r �_newest_cmtimerJ rQ �files_cache_loggerr# r1 r2 r3 r� ra r� r� r ZUnpackerrF �feedr* �packb�_replacer� � TypeError�
ValueErrorrG rI r% rK �len)
r7 rb rM �u�data� path_hash�item�entryrO �fier8 r8 r9 r� s<
& &
zLocalCache._read_filesc C s t dd�}tjj| jd�}tj|� |jd� tjtjj| jd�|� |jd� tjtjj| jd�|� |jd� y tjtjj| jt� �|� W n8 t k
r� t
tjj|t� �d d
�� W d Q R X Y nX tj tjj| jd�tjj| jd �� d | _ |j
� d S ) Nzcache.begin_transaction)�msgidztxn.tmpz.Initializing cache transaction: Reading configr� z.Initializing cache transaction: Reading chunksr� z-Initializing cache transaction: Reading filesT)r� z
txn.active)r r1 r2 r3 �mkdir�outputr< �copyr� r\ r' �renamer� �finish)r7 �pi�txn_dirr8 r8 r9 � begin_txn, s"
zLocalCache.begin_txnc C s� | j s
dS | jj| j| j� tdd�}| jdk �r4| jdkrBd| _tt j
j dd��}|j d � t
jd
� tt jj| jt� �d d ��v}d
}xj| jj� D ]\\}}ttj|�� }|jd
kr�t|j�| jk s�|jd
kr�|j|k r�tj||f|� |d7 }q�W W dQ R X t
jd|� t
jd| j� t
jd|� |j| j j t� < |j d� tt jj| jd�d d ��}| j j |� W dQ R X |j| j j d< |j d� | j j| j| j� t j t jj| jd�t jj| jd�� t!j"t jj| jd�� d| _ |j#� dS )zCommit transaction
Nz cache.commit)r� � �? r ZBORG_FILES_CACHE_TTL� zSaving files cachez FILES-CACHE-SAVE: starting...T)r2 rU r z>FILES-CACHE-KILL: removed all old entries with age >= TTL [%d]zCFILES-CACHE-KILL: removed all current entries with newest cmtime %dz7FILES-CACHE-SAVE: finished, %d remaining entries saved.zSaving chunks cacher� zSaving cache configz
txn.activeztxn.tmpFl l ���� )$r� r� rW rV rL r r r� �intr1 r� r� r� r� rQ r# r2 r3 r� r� r* r �unpackbr� r �cmtime�packr� ra r� r� rU r� r< r= r� )r7 r� ZttlrM Z entry_countr� r� r� r8 r8 r9 r� @ sB
zLocalCache.commitc C s� t jjt jj| jd��r.tjt jj| jd�� t jj| jd�}t jj|�r�tjt jj|d�| j� tjt jj|d�| j� tjt jj|t|��| j� t j|t jj| jd�� t jjt jj| jd��r�tjt jj| jd�� d| _ | j
� dS )z3Roll back partial and aborted transactions
ztxn.tmpz
txn.activer� r� FN) r1 r2 r; r3 r< r= r� r� r� r� r� )r7 r� r8 r8 r9 r� i s zLocalCache.rollbackc sJ t jj�jd�� t� �d�
d�d�d�
d� d � fdd� � � �fdd���fdd ���fd
d ��d � fd
d� ���� �
fdd����
���fdd���� fdd���� �fdd���fdd�� ������� �
� �
����fdd�}� �fdd �}�jjtjjf� �j � t
�j �j d ��&�|� t jj
� ��_|�j��_W d Q R X d S )!a� Re-synchronize chunks cache with repository.
Maintains a directory with known backup archive indexes, so it only
needs to fetch infos from repo and build a chunk index once per backup
archive.
If out of sync, missing archive indexes get added, outdated indexes
get removed and a new master chunks index is built by merging all
archive indexes.
zchunks.archive.dr rd c s t | �}tjj� || �}|S )N)r r1 r2 r3 )r� r� Zid_hexr2 )� archive_pathr8 r9 �mkpath� s z LocalCache.sync.<locals>.mkpathc s> �j r4tj� �} tdd� | D ��tdd� | D ��B S t� S d S )Nc s s" | ]}t |�d krt|�V qdS )r� N)r� r )r? r� r8 r8 r9 rA � s z;LocalCache.sync.<locals>.cached_archives.<locals>.<genexpr>c s s4 | ],}t |�d kr|jd�rt|dd� �V qdS )�H z.compactNr� )r� �endswithr )r? r� r8 r8 r9 rA � s )�do_cacher1 r� r� )Zfns)r� r7 r8 r9 �cached_archives� s
z(LocalCache.sync.<locals>.cached_archivesc s t dd� � jjj� D ��S )Nc s s | ] }|j V qd S )N)r� )r? �infor8 r8 r9 rA � s z9LocalCache.sync.<locals>.repo_archives.<locals>.<genexpr>)r� rV �archives�listr8 )r7 r8 r9 �
repo_archives� s z&LocalCache.sync.<locals>.repo_archivesc s x| D ] }� |� qW d S )Nr8 )�idsr� )�cleanup_cached_archiver8 r9 �cleanup_outdated� s
z)LocalCache.sync.<locals>.cleanup_outdatedTc s� y$t j� | �� t j� | �d � W n tk
r8 Y nX |sBd S y,t j� | dd�� t j� | dd�d � W n tk
r� Y nX d S )Nz
.integrityz.compact)r� )r1 �unlinkr\ )r� �cleanup_compact)r� r8 r9 r� � s z/LocalCache.sync.<locals>.cleanup_cached_archivec s� | j � }g }t� �rlxZ|D ]L}� j|�}|r\| | j|jd�}|j|jksRtd��|| |< q|j|� qW n|}xVt|�j j
|��D ]@\}}| | jt|�d�}|| |< |� |< �d7 ��t|�7 �q�W dS )z�
Archives created with AdHocCache will have csize=0 in all chunk list entries whose
chunks were already in the repository.
Scan *chunk_idx* for entries where csize=0 and fill in the correct information.
)�csizezChunk size mismatchr N) Zzero_csize_idsr� r� r� r �sizer� �append�zipr- �get_many)� chunk_idxZall_missing_idsZ fetch_ids�id_Zalready_fetched_entryr� r� )�chunks_fetched_size_index�decrypted_repository�fetched_bytes_for_csize�fetched_chunks_for_csizer8 r9 �fetch_missing_csize� s"
z,LocalCache.sync.<locals>.fetch_missing_csizec
s� |j | �\}}|j| dt|�|� �jj|dd�\}}}t|d�}|jdkrTtd��t|�}xVt |j
|j |j
��D ]>\} \}}|j| dt|�|� �t|�7 ��d7 �|j |� qrW �j
r̈ |� �| |� d S )Nr T)Zforce_tam_not_required)Z
internal_dictz Unknown archive metadata version)r� �addr� rL Zunpack_and_verify_archiver r� r� r r r� r r� r� )
�
archive_idr r r r� �archiveZverified�_r� Zitem_id)r � processed_item_metadata_bytes� processed_item_metadata_chunksr7 �write_archive_indexr8 r9 �fetch_and_build_idx� s
" z,LocalCache.sync.<locals>.fetch_and_build_idxc s� � |j � 7 � �| dd�}�| dd�}y0t|dt| �d d��}|j|� W d Q R X W n tk
rp t|� Y nX tj||� d S )Nz.compact)r� z.tmpT)r2 rU �filename)Zcompactr$ r rU r� r r1 r� )r r r� Zfn_tmprM )�"compact_chunks_archive_saved_spacer� r8 r9 r � s z,LocalCache.sync.<locals>.write_archive_indexc s� �| �}t jd|� yvy:t|d dd��}tj|dd�}W d Q R X � | dd� |S tk
r� t|dd��}tj|�}W d Q R X Y nX W n> tk
r� } z"t jd||� � | � tt � d S d }~X nX t j
d |� � | � �| |� |S )
Nz-Reading cached archive chunk index for %s ...z.compactF)r2 rU T)Zpermit_compact)r z1Cached archive chunk index of %s is corrupted: %sz6Found non-compact index for %s, converting to compact.) rJ r� r$ r rF r\ r% �errorr r rQ )r � archive_nameZarchive_chunk_idx_pathrM �archive_chunk_idxr� )r� r� r r8 r9 �read_archive_index� s(
z+LocalCache.sync.<locals>.read_archive_indexc sH i }x*� j jj� D ]}|j| kr|j||j< qW t|�t| �ksDt�|S )N)rV r� r� r� �namer� r� )� archive_idsZ
archive_namesr� )r7 r8 r9 �get_archive_ids_to_names s
z1LocalCache.sync.<locals>.get_archive_ids_to_namesc s� t jd� � � }� � }t jdt|�t|�t|| �t|| �� | j� �|| � tt�
j�tj �}|�r��
jsxd nt|�} t t|�dddd�}�|�}x�|j
� D ]�\}}|j t |�gd� �
j�r*||kr� ||�}|d kr�|j
|� ||k�rt jd|� t� }�|�|� t jd � | j|� q�| �p6t|�} t jd
|� �|�| � q�W �
j�sd�| � |j� t jd t���� t jd t� ��
� t jd
t��� t jd� | S )Nz Synchronizing chunks cache...zIArchives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.g�������?z3%3.0f%% Syncing chunks cache. Processing archive %sz
cache.sync)�total�steprb r� )r� z.Fetching and building archive index for %s ...z$Merging into master chunks index ...z!Fetching archive index for %s ...zlCache sync: had to fetch %s (%d chunks) because no archive had a csize set for them (due to --no-cache-sync)z0Cache sync: processed %s (%d chunks) of metadataz;Cache sync: compact chunks.archive.d storage saved %s byteszDone.)rJ r� r� �clearr� r- r �MAX_LOAD_FACTORr� r r� �showr r� �merger� rQ r ) r Z
cached_idsr Zmaster_index_capacityr� Zarchive_ids_to_namesr r r )r� r r r r r r r
r r r r r� r7 r8 r9 �create_master_idx sT
z*LocalCache.sync.<locals>.create_master_idxc sp yt jt jj�jd�� W n Y nX yt jt jj�jd�� W n Y nX yt j� � W n Y nX dS )z?bring old cache dirs into the desired state (cleanup and adapt)zchunks.archivezchunks.archive.tmpN)r1 r r2 r3 r� r8 )r� r7 r8 r9 �legacy_cleanupQ s z'LocalCache.sync.<locals>.legacy_cleanup)Zdecrypted_cacheN)rd )T)r1 r2 r3 r rV Z check_repository_compatibilityr Z OperationZREADr� r( r- rL r� r� r� )r7 r& r' r8 )r� r� r
r� r r r r r r r
r r� r r r r� r7 r r9 r� { s2
! &3zLocalCache.syncc C s* t j}| jj|@ rdS | jj|ks&dS dS )NFT)r �SUPPORTED_REPO_FEATURESra r� r� )r7 � my_featuresr8 r8 r9 r� m s z$LocalCache.check_cache_compatibilityc C s� t jd� tjj| jd�}tjj|�rRtjtjj| jd�� tjtjj| jd�� t � | _
t tjj| jt � �dd�� W d Q R X d| j
_| j
jjddd� t� | j
_t� | j
_d S )Nz9Discarding incompatible cache and forcing a cache rebuildzchunks.archive.dT)r� rd r� rV )rJ rK r1 r2 r3 r� r< r= r� r r� r' r� ra r� r� r� r� r� )r7 r� r8 r8 r9 r� y s
zLocalCache.wipe_cachec C s^ | j j� }tj}t� }x |j� D ]\}}|j|� q W | jjj|| � | jj j||@ � d S )N)
rV Zget_all_mandatory_featuresr r( r� r� �updatera r� r� )r7 Zoperation_to_features_mapr) Z
repo_featuresZ operationZfeaturesr8 r8 r9 r� � s
z LocalCache.update_compatibilityc
C s� | j s| j� t|�}| j||�}|r8| r8| j||�S | jj|�}t|�} | jj|||d� | j j
|d|| � |j || | � t ||| �S )N)�waitr )
r� r� r� �
seen_chunk� chunk_increfrL �encryptr- �putr� r r* r! )
r7 r� �chunkr� � overwriter+ r �refcountr� r r8 r8 r9 � add_chunk� s
zLocalCache.add_chunkc C sJ | j j|tdd d ��\}}}|d k rF|d k rF||krFtd|||f ��|S )Nr z@chunk has same id [%r], but different size (stored: %d new: %d)!)r� r� r
r� )r7 r� r r2 Z stored_sizer r8 r8 r9 r, � s
zLocalCache.seen_chunkc C s: | j s| j� | jj|�\}}}|j||d� t|||�S )NF)r� r� r� �increfr* r! )r7 r� r� r �count�_sizer r8 r8 r9 r- � s
zLocalCache.chunk_increfc C sj | j s| j� | jj|�\}}}|dkrT| j|= | jj||d� |j| | d� n|j| | d� d S )Nr )r+ TF)r� r� r� �decrefr- �deleter* )r7 r� r� r+ r5 r r r8 r8 r9 � chunk_decref� s zLocalCache.chunk_decrefc C s4 t j|j�sdS | j}d|kr,tjd� dS d|krBtjd� dS | jj|�}|sbtjd|� dS tt j
|�� }d|kr�|j |j kr�tjd |� dS d |kr�|j
|jkr�tjd |� dS d
|ko�t|j�|jk� r�tjd|� dS d|k�rt|j�|jk�rtjd|� dS t j|j|jdd��| j|< d
|jfS )aj
Check if we know the file that has this path_hash (know == it is in our files cache) and
whether it is unchanged (the size/inode number/cmtime is same for stuff we check in this cache_mode).
:param hashed_path: the file's path as we gave it to hash(hashed_path)
:param path_hash: hash(hashed_path), to save some memory in the files cache
:param st: the file's stat() result
:return: known, ids (known is True if we have infos about this file in the cache,
ids is the list of chunk ids IF the file has not changed, otherwise None).
FNr� z UNKNOWN: files cache disabledrD z UNKNOWN: rechunking enforcedz*UNKNOWN: no file metadata in cache for: %r�sz(KNOWN-CHANGED: file size has changed: %rTr� z0KNOWN-CHANGED: file inode number has changed: %r�cz)KNOWN-CHANGED: file ctime has changed: %r�mz)KNOWN-CHANGED: file mtime has changed: %rr )�inoder� )FN)FN)FN)FN)TN)TN)TN)TN)�stat�S_ISREG�st_moder� r� rQ r r� r* r r� r �st_sizer= �st_inor r� � st_ctime_ns� st_mtime_nsr� r� � chunk_ids)r7 � hashed_pathr� �str� r� r8 r8 r9 �file_known_and_unchanged� s8
z#LocalCache.file_known_and_unchangedc C s� t j|j�sd S | j}d|kr,tjd� d S d|krDd}t|j�}nd|krZd}t|j�}t d|j
|j t |�|d�}t
j|�| j|< t| jp�d|�| _tjd |jd
t|j� d �||� d S ) Nr� z*FILES-CACHE-NOUPDATE: files cache disabledr; Zctimer< �mtimer )r� r= r r� rE z)FILES-CACHE-UPDATE: put %r [has %s] <- %rz [%d entries])rE )r> r? r@ r� r� rQ r rC rD r* rB rA r r r� r re r� r� r� rE ) r7 rF r� rG r� r� Z cmtime_typeZ cmtime_nsr� r8 r8 r9 �
memorize_file� s$
zLocalCache.memorize_file)FT)N)N)T)rt ru rv rw r r: r� r� r� r� rE r� r� r� r� r� r� r� r� r� r3 r, r- r9 rH rJ r8 r8 r8 r9 r� � s0 (
!) s
4r� c @ s� e Zd ZdZdZd dd�Zdd� Zd d
� ZdZd Z d d
� Z
dd� Z d dd�Z d!dd�Z
d"dd�Zd#dd�Zdd� Zdd � Zd d � ZdS )$r� a�
Ad-hoc, non-persistent cache.
Compared to the standard LocalCache the AdHocCache does not maintain accurate reference count,
nor does it provide a files cache (which would require persistence). Chunks that were not added
during the current AdHocCache lifetime won't have correct size/csize set (0 bytes) and will
have an infinite reference count (MAX_VALUE).
z�All archives: unknown unknown unknown
Unique chunks Total chunks
Chunk index: {0.total_unique_chunks:20d} unknownTNc C sB || _ || _|| _d| _t|�| _| jj|||d� tjd� d S )NF)rn z1Note: --no-cache-sync is an experimental feature.) r- rL rV � _txn_activer+ r� rr rJ rK )r7 r- rL rV rm rn r8 r8 r9 r: s
zAdHocCache.__init__c C s | S )Nr8 )r7 r8 r8 r9 r� " s zAdHocCache.__enter__c C s d S )Nr8 )r7 r� r� r� r8 r8 r9 r� % s zAdHocCache.__exit__r� c C s t jd� dS )Nz$UNKNOWN: files cache not implementedF)FN)r� rQ )r7 rF r� rG r8 r8 r9 rH + s
z#AdHocCache.file_known_and_unchangedc C s d S )Nr8 )r7 rF r� rG r� r8 r8 r9 rJ / s zAdHocCache.memorize_fileFc
C s� | st d��| js | j� t|�}| j||�}|rD| j|||d�S | jj|�}t|�} | jj |||d� | j
j |d|| � |j || | � t
||| �S )NuH AdHocCache does not permit overwrites — trying to use it for recreate?)r )r+ r )r� rK �
_begin_txnr� r, r- rL r. r- r/ r� r r* r! )
r7 r� r0 r� r1 r+ r r2 r� r r8 r8 r9 r3 2 s zAdHocCache.add_chunkc C sN | j s| j� | jj|tdd d ��}|jrH|rH|j rH|j|d�| j|< |jS )Nr )r )rK rL r� r� r
r2 r r� )r7 r� r r� r8 r8 r9 r, A s zAdHocCache.seen_chunkc C sJ | j s| j� | jj|�\}}}|p&|}|s0t�|j||d� t|||�S )NF)rK rL r� r4 r� r* r! )r7 r� r� r r5 r6 r r8 r8 r9 r- L s zAdHocCache.chunk_increfc C sj | j s| j� | jj|�\}}}|dkrT| j|= | jj||d� |j| | d� n|j| | d� d S )Nr )r+ TF)rK rL r� r7 r- r8 r* )r7 r� r� r+ r5 r r r8 r8 r9 r9 W s zAdHocCache.chunk_decrefc C s& | j s
d S | jj| j| j� d| _ d S )NF)rK r� rW rV rL )r7 r8 r8 r9 r� b s zAdHocCache.commitc C s d| _ | `d S )NF)rK r� )r7 r8 r8 r9 r� h s zAdHocCache.rollbackc C s d| _ t| j�}t|tj d �}t|�| _t|ddd�}t� }d}d }xb| jj t
|d�}|d7 }|shP |j t|�d � |d }t tj
ddd
�}x|D ]} || j| < q�W qJW t| j�|ks�t�| j| jj= t� | p�d }
|j� tjd ||
|t|d
|
�� d S )NTg�������?z!Downloading chunk list... %3.0f%%zcache.download_chunks)r rb r� r )�limit�markerr )Zincrease)r2 r r g{�G�z�?zBAdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s�" ���)rK r� r- r� r r# r� r r r� r) r$ r
Z MAX_VALUEr� rV Z MANIFEST_IDr� rJ rQ r ) r7 Z
num_chunksZcapacityr� Zt0Z num_requestsrN �resultZ
init_entryr Zdurationr8 r8 r9 rL n s2
zAdHocCache._begin_txn)TN)FT)N)N)T)rt ru rv rw r� r: r� r� r r� rH rJ r3 r, r- r9 r� r� rL r8 r8 r8 r9 r� s
r� )N)Fr� r1 r< r> Zbinasciir � collectionsr Ztimer rJ r r� Z constantsr r Z hashindexr r
r Zhelpersr r
r r r r r r r r r r r r r r r r r r r� r r! Z
crypto.keyr" Zcrypto.file_integrityr# r$ r% Zlockingr& �platformr' Zremoter( r- r) r* r+ rr r~ r0 r� r� rq r_ r� r� r� r8 r8 r8 r9 �<module> s`
%
eE S