XAR (eXtensible ARchiver) Format

Overview

XAR (eXtensible ARchiver) is an open-source archival format developed by Apple for creating compressed archive files. It provides superior compression, integrity checking, and extensibility compared to traditional archive formats like TAR and ZIP.

Technical Specifications

• Format Type: Archive container
• File Extension: .xar
• MIME Type: application/x-xar
• Compression: Gzip, Bzip2, LZMA support
• Checksum: SHA-1, MD5 support
• Metadata: XML-based table of contents
• Endianness: Big-endian format

Format Structure

XAR archives consist of:

• Fixed-size header with magic number and metadata
• XML table of contents (TOC) describing archive structure
• Compressed file data heap
• Checksum information for integrity verification
• Extended attributes and metadata support

History and Development

• 2005: Developed by Apple for Mac OS X installers
• 2006: Open-sourced under BSD license
• 2007: Used in Mac OS X package installers (.pkg)
• 2010: Adoption in other Unix-like systems
• Present: Maintained as open-source project

Use Cases

• Mac OS X package installation files
• Software distribution archives
• Backup and archival solutions
• Cross-platform archive distribution
• Applications requiring integrity verification
• Systems with advanced metadata requirements

Code Examples

C/C++ XAR Creation and Extraction

#include <xar/xar.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

// Create XAR archive
int create_xar_archive(const char* archive_path, const char** files, int file_count) {
    xar_t archive;
    xar_file_t file;
    xar_stream stream;
    
    // Create new archive
    archive = xar_open(archive_path, WRITE);
    if (!archive) {
        fprintf(stderr, "Failed to create archive: %s\n", archive_path);
        return -1;
    }
    
    // Set compression algorithm
    xar_opt_set(archive, XAR_OPT_COMPRESSION, XAR_OPT_VAL_GZIP);
    
    // Add files to archive
    for (int i = 0; i < file_count; i++) {
        FILE* input = fopen(files[i], "rb");
        if (!input) {
            fprintf(stderr, "Cannot open file: %s\n", files[i]);
            continue;
        }
        
        // Add file to archive
        file = xar_add(archive, files[i]);
        if (!file) {
            fprintf(stderr, "Failed to add file: %s\n", files[i]);
            fclose(input);
            continue;
        }
        
        // Copy file data
        char buffer[8192];
        size_t bytes_read;
        
        while ((bytes_read = fread(buffer, 1, sizeof(buffer), input)) > 0) {
            if (xar_write(file, buffer, bytes_read) != bytes_read) {
                fprintf(stderr, "Write error for file: %s\n", files[i]);
                break;
            }
        }
        
        fclose(input);
        printf("Added: %s\n", files[i]);
    }
    
    // Close archive
    xar_close(archive);
    printf("Archive created: %s\n", archive_path);
    return 0;
}

// Extract XAR archive
int extract_xar_archive(const char* archive_path, const char* extract_dir) {
    xar_t archive;
    xar_iter_t iter;
    xar_file_t file;
    char* filename;
    
    // Open archive for reading
    archive = xar_open(archive_path, READ);
    if (!archive) {
        fprintf(stderr, "Failed to open archive: %s\n", archive_path);
        return -1;
    }
    
    // Iterate through files
    iter = xar_iter_new();
    file = xar_file_first(archive, iter);
    
    while (file) {
        // Get filename
        if (xar_get_path(file, &filename) != 0) {
            fprintf(stderr, "Failed to get filename\n");
            file = xar_file_next(iter);
            continue;
        }
        
        // Create full extract path
        char extract_path[1024];
        snprintf(extract_path, sizeof(extract_path), "%s/%s", extract_dir, filename);
        
        // Extract file
        if (xar_extract_tofile(archive, file, extract_path) != 0) {
            fprintf(stderr, "Failed to extract: %s\n", filename);
        } else {
            printf("Extracted: %s\n", filename);
        }
        
        free(filename);
        file = xar_file_next(iter);
    }
    
    xar_iter_free(iter);
    xar_close(archive);
    printf("Extraction complete\n");
    return 0;
}

// List archive contents
int list_xar_contents(const char* archive_path) {
    xar_t archive;
    xar_iter_t iter;
    xar_file_t file;
    char* filename;
    const char* size_str;
    const char* type_str;
    
    archive = xar_open(archive_path, READ);
    if (!archive) {
        fprintf(stderr, "Failed to open archive: %s\n", archive_path);
        return -1;
    }
    
    printf("Contents of %s:\n", archive_path);
    printf("%-50s %10s %10s\n", "Filename", "Size", "Type");
    printf("%-50s %10s %10s\n", "--------", "----", "----");
    
    iter = xar_iter_new();
    file = xar_file_first(archive, iter);
    
    while (file) {
        xar_get_path(file, &filename);
        xar_get_size(file, &size_str);
        xar_get_type(file, &type_str);
        
        printf("%-50s %10s %10s\n", filename, size_str, type_str);
        
        free(filename);
        file = xar_file_next(iter);
    }
    
    xar_iter_free(iter);
    xar_close(archive);
    return 0;
}

// Example usage
int main(int argc, char* argv[]) {
    if (argc < 3) {
        printf("Usage: %s <command> <archive> [files...]\n", argv[0]);
        printf("Commands: create, extract, list\n");
        return 1;
    }
    
    const char* command = argv[1];
    const char* archive = argv[2];
    
    if (strcmp(command, "create") == 0) {
        if (argc < 4) {
            printf("Usage: %s create <archive> <file1> [file2...]\n", argv[0]);
            return 1;
        }
        return create_xar_archive(archive, (const char**)&argv[3], argc - 3);
    }
    else if (strcmp(command, "extract") == 0) {
        const char* extract_dir = argc > 3 ? argv[3] : ".";
        return extract_xar_archive(archive, extract_dir);
    }
    else if (strcmp(command, "list") == 0) {
        return list_xar_contents(archive);
    }
    else {
        printf("Unknown command: %s\n", command);
        return 1;
    }
}

Python XAR Processing

import subprocess
import xml.etree.ElementTree as ET
import os
import tempfile
import hashlib

class XarArchive:
    def __init__(self, archive_path):
        self.archive_path = archive_path
        self._check_xar_availability()
    
    def _check_xar_availability(self):
        """Check if xar command is available."""
        try:
            subprocess.run(['xar', '--version'], 
                         capture_output=True, check=True)
        except (subprocess.CalledProcessError, FileNotFoundError):
            raise RuntimeError("XAR command not found. Please install libxar.")
    
    def create(self, files, compression='gzip'):
        """Create XAR archive from list of files."""
        cmd = ['xar', '-cf', self.archive_path]
        
        # Set compression
        if compression:
            cmd.extend(['--compression', compression])
        
        # Add files
        cmd.extend(files)
        
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
            print(f"Archive created: {self.archive_path}")
            return True
        except subprocess.CalledProcessError as e:
            print(f"Failed to create archive: {e.stderr}")
            return False
    
    def extract(self, extract_dir='.'):
        """Extract XAR archive to specified directory."""
        cmd = ['xar', '-xf', self.archive_path, '-C', extract_dir]
        
        try:
            subprocess.run(cmd, check=True)
            print(f"Archive extracted to: {extract_dir}")
            return True
        except subprocess.CalledProcessError as e:
            print(f"Failed to extract archive: {e}")
            return False
    
    def list_contents(self):
        """List contents of XAR archive."""
        cmd = ['xar', '-tf', self.archive_path]
        
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
            files = result.stdout.strip().split('\n')
            return [f for f in files if f]
        except subprocess.CalledProcessError as e:
            print(f"Failed to list archive: {e.stderr}")
            return []
    
    def get_toc(self):
        """Extract and parse Table of Contents."""
        cmd = ['xar', '--dump-toc=-', '-f', self.archive_path]
        
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
            return ET.fromstring(result.stdout)
        except subprocess.CalledProcessError as e:
            print(f"Failed to get TOC: {e.stderr}")
            return None
        except ET.ParseError as e:
            print(f"Failed to parse TOC XML: {e}")
            return None
    
    def verify_integrity(self):
        """Verify archive integrity using checksums."""
        toc = self.get_toc()
        if toc is None:
            return False
        
        # Extract to temporary directory for verification
        with tempfile.TemporaryDirectory() as temp_dir:
            if not self.extract(temp_dir):
                return False
            
            # Verify each file
            for file_elem in toc.findall('.//file'):
                name_elem = file_elem.find('name')
                checksum_elem = file_elem.find('.//extracted-checksum')
                
                if name_elem is None or checksum_elem is None:
                    continue
                
                filename = name_elem.text
                expected_checksum = checksum_elem.text
                checksum_style = checksum_elem.get('style', 'sha1')
                
                file_path = os.path.join(temp_dir, filename)
                if not os.path.exists(file_path):
                    print(f"File not found: {filename}")
                    return False
                
                # Calculate checksum
                actual_checksum = self._calculate_checksum(file_path, checksum_style)
                
                if actual_checksum != expected_checksum:
                    print(f"Checksum mismatch for {filename}")
                    print(f"Expected: {expected_checksum}")
                    print(f"Actual: {actual_checksum}")
                    return False
        
        print("Archive integrity verified")
        return True
    
    def _calculate_checksum(self, file_path, algorithm):
        """Calculate file checksum."""
        if algorithm.lower() == 'sha1':
            hasher = hashlib.sha1()
        elif algorithm.lower() == 'md5':
            hasher = hashlib.md5()
        else:
            raise ValueError(f"Unsupported checksum algorithm: {algorithm}")
        
        with open(file_path, 'rb') as f:
            for chunk in iter(lambda: f.read(4096), b""):
                hasher.update(chunk)
        
        return hasher.hexdigest()
    
    def get_metadata(self):
        """Extract metadata from archive."""
        toc = self.get_toc()
        if toc is None:
            return {}
        
        metadata = {
            'files': [],
            'total_size': 0,
            'compression': None
        }
        
        # Parse creation time
        creation_time = toc.find('.//creation-time')
        if creation_time is not None:
            metadata['creation_time'] = creation_time.text
        
        # Parse file information
        for file_elem in toc.findall('.//file'):
            file_info = {}
            
            for child in file_elem:
                if child.tag in ['name', 'type', 'mode', 'uid', 'gid', 'user', 'group']:
                    file_info[child.tag] = child.text
                elif child.tag == 'data':
                    size_elem = child.find('size')
                    if size_elem is not None:
                        file_info['size'] = int(size_elem.text)
                        metadata['total_size'] += file_info['size']
                    
                    encoding_elem = child.find('encoding')
                    if encoding_elem is not None:
                        file_info['compression'] = encoding_elem.get('style')
                        if not metadata['compression']:
                            metadata['compression'] = file_info['compression']
            
            metadata['files'].append(file_info)
        
        return metadata

# Usage examples
def create_sample_archive():
    """Create a sample XAR archive."""
    # Create some sample files
    sample_files = ['file1.txt', 'file2.txt', 'file3.txt']
    
    for i, filename in enumerate(sample_files):
        with open(filename, 'w') as f:
            f.write(f"This is sample file {i+1}\n" * 100)
    
    # Create archive
    archive = XarArchive('sample.xar')
    success = archive.create(sample_files, compression='gzip')
    
    # Clean up sample files
    for filename in sample_files:
        os.remove(filename)
    
    return success

def analyze_archive(archive_path):
    """Analyze XAR archive and display information."""
    archive = XarArchive(archive_path)
    
    print(f"Analyzing archive: {archive_path}")
    print("=" * 50)
    
    # List contents
    files = archive.list_contents()
    print(f"Files in archive: {len(files)}")
    for file in files:
        print(f"  - {file}")
    
    # Get metadata
    metadata = archive.get_metadata()
    print(f"\nTotal size: {metadata.get('total_size', 0)} bytes")
    print(f"Compression: {metadata.get('compression', 'none')}")
    print(f"Creation time: {metadata.get('creation_time', 'unknown')}")
    
    # Verify integrity
    print(f"\nIntegrity check: {'PASSED' if archive.verify_integrity() else 'FAILED'}")

# Command-line interface
if __name__ == "__main__":
    import sys
    
    if len(sys.argv) < 2:
        print("Usage: python xar_processor.py <command> [args...]")
        print("Commands:")
        print("  create <archive> <file1> [file2...]")
        print("  extract <archive> [extract_dir]")
        print("  list <archive>")
        print("  analyze <archive>")
        print("  verify <archive>")
        sys.exit(1)
    
    command = sys.argv[1]
    
    if command == "create":
        if len(sys.argv) < 4:
            print("Usage: python xar_processor.py create <archive> <file1> [file2...]")
            sys.exit(1)
        
        archive_path = sys.argv[2]
        files = sys.argv[3:]
        
        archive = XarArchive(archive_path)
        archive.create(files)
    
    elif command == "extract":
        if len(sys.argv) < 3:
            print("Usage: python xar_processor.py extract <archive> [extract_dir]")
            sys.exit(1)
        
        archive_path = sys.argv[2]
        extract_dir = sys.argv[3] if len(sys.argv) > 3 else '.'
        
        archive = XarArchive(archive_path)
        archive.extract(extract_dir)
    
    elif command == "list":
        if len(sys.argv) < 3:
            print("Usage: python xar_processor.py list <archive>")
            sys.exit(1)
        
        archive_path = sys.argv[2]
        archive = XarArchive(archive_path)
        
        files = archive.list_contents()
        print(f"Files in {archive_path}:")
        for file in files:
            print(f"  {file}")
    
    elif command == "analyze":
        if len(sys.argv) < 3:
            print("Usage: python xar_processor.py analyze <archive>")
            sys.exit(1)
        
        analyze_archive(sys.argv[2])
    
    elif command == "verify":
        if len(sys.argv) < 3:
            print("Usage: python xar_processor.py verify <archive>")
            sys.exit(1)
        
        archive = XarArchive(sys.argv[2])
        if archive.verify_integrity():
            print("Archive integrity: PASSED")
        else:
            print("Archive integrity: FAILED")
    
    else:
        print(f"Unknown command: {command}")
        sys.exit(1)

Shell Scripts for XAR Operations

#!/bin/bash

# XAR archive management script

# Function to create XAR archive with options
create_xar_archive() {
    local archive_name="$1"
    local compression="$2"
    shift 2
    local files=("$@")
    
    if [ ${#files[@]} -eq 0 ]; then
        echo "Error: No files specified"
        return 1
    fi
    
    echo "Creating XAR archive: $archive_name"
    echo "Compression: $compression"
    echo "Files: ${files[*]}"
    
    xar -cf "$archive_name" --compression="$compression" "${files[@]}"
    
    if [ $? -eq 0 ]; then
        echo "Archive created successfully"
        echo "Archive size: $(ls -lh "$archive_name" | awk '{print $5}')"
    else
        echo "Failed to create archive"
        return 1
    fi
}

# Function to extract XAR archive
extract_xar_archive() {
    local archive_name="$1"
    local extract_dir="${2:-.}"
    
    if [ ! -f "$archive_name" ]; then
        echo "Error: Archive file not found: $archive_name"
        return 1
    fi
    
    mkdir -p "$extract_dir"
    
    echo "Extracting $archive_name to $extract_dir"
    xar -xf "$archive_name" -C "$extract_dir"
    
    if [ $? -eq 0 ]; then
        echo "Extraction completed"
        echo "Extracted files:"
        xar -tf "$archive_name"
    else
        echo "Failed to extract archive"
        return 1
    fi
}

# Function to list archive contents with details
list_archive_details() {
    local archive_name="$1"
    
    if [ ! -f "$archive_name" ]; then
        echo "Error: Archive file not found: $archive_name"
        return 1
    fi
    
    echo "Archive: $archive_name"
    echo "Size: $(ls -lh "$archive_name" | awk '{print $5}')"
    echo ""
    echo "Contents:"
    xar -tvf "$archive_name"
    
    echo ""
    echo "Table of Contents (XML):"
    xar --dump-toc=- -f "$archive_name" | head -20
}

# Function to verify archive integrity
verify_archive() {
    local archive_name="$1"
    
    if [ ! -f "$archive_name" ]; then
        echo "Error: Archive file not found: $archive_name"
        return 1
    fi
    
    echo "Verifying archive integrity: $archive_name"
    
    # Create temporary directory
    local temp_dir=$(mktemp -d)
    
    # Extract archive
    if xar -xf "$archive_name" -C "$temp_dir" >/dev/null 2>&1; then
        echo "Archive extraction: OK"
        
        # Get TOC and verify checksums
        local toc_file=$(mktemp)
        xar --dump-toc="$toc_file" -f "$archive_name"
        
        # Parse and verify each file (simplified check)
        local files_count=$(xar -tf "$archive_name" | wc -l)
        local extracted_count=$(find "$temp_dir" -type f | wc -l)
        
        if [ "$files_count" -eq "$extracted_count" ]; then
            echo "File count verification: OK ($files_count files)"
            echo "Archive integrity: PASSED"
        else
            echo "File count mismatch: expected $files_count, found $extracted_count"
            echo "Archive integrity: FAILED"
        fi
        
        rm -f "$toc_file"
    else
        echo "Archive extraction: FAILED"
        echo "Archive integrity: FAILED"
    fi
    
    # Cleanup
    rm -rf "$temp_dir"
}

# Function to compare compression algorithms
compare_compressions() {
    local input_files=("$@")
    local compressions=("none" "gzip" "bzip2")
    
    if [ ${#input_files[@]} -eq 0 ]; then
        echo "Error: No input files specified"
        return 1
    fi
    
    echo "Comparing compression algorithms for files: ${input_files[*]}"
    echo ""
    
    for compression in "${compressions[@]}"; do
        local archive_name="test_${compression}.xar"
        
        echo "Testing $compression compression..."
        
        # Create archive
        local start_time=$(date +%s.%N)
        xar -cf "$archive_name" --compression="$compression" "${input_files[@]}" 2>/dev/null
        local end_time=$(date +%s.%N)
        
        if [ -f "$archive_name" ]; then
            local archive_size=$(stat -f%z "$archive_name" 2>/dev/null || stat -c%s "$archive_name")
            local creation_time=$(echo "$end_time - $start_time" | bc)
            
            printf "  %-10s: %10s bytes, %.3f seconds\n" "$compression" "$archive_size" "$creation_time"
            
            # Clean up
            rm -f "$archive_name"
        else
            echo "  $compression: FAILED"
        fi
    done
}

# Main script logic
case "$1" in
    "create")
        if [ $# -lt 4 ]; then
            echo "Usage: $0 create <archive> <compression> <file1> [file2...]"
            echo "Compression options: none, gzip, bzip2"
            exit 1
        fi
        create_xar_archive "$2" "$3" "${@:4}"
        ;;
    
    "extract")
        if [ $# -lt 2 ]; then
            echo "Usage: $0 extract <archive> [extract_dir]"
            exit 1
        fi
        extract_xar_archive "$2" "$3"
        ;;
    
    "list")
        if [ $# -lt 2 ]; then
            echo "Usage: $0 list <archive>"
            exit 1
        fi
        list_archive_details "$2"
        ;;
    
    "verify")
        if [ $# -lt 2 ]; then
            echo "Usage: $0 verify <archive>"
            exit 1
        fi
        verify_archive "$2"
        ;;
    
    "compare")
        if [ $# -lt 2 ]; then
            echo "Usage: $0 compare <file1> [file2...]"
            exit 1
        fi
        compare_compressions "${@:2}"
        ;;
    
    *)
        echo "Usage: $0 {create|extract|list|verify|compare} [options...]"
        echo ""
        echo "Commands:"
        echo "  create <archive> <compression> <file1> [file2...]"
        echo "  extract <archive> [extract_dir]"
        echo "  list <archive>"
        echo "  verify <archive>"
        echo "  compare <file1> [file2...]"
        exit 1
        ;;
esac

Installation and Setup

macOS (Built-in)

# XAR is included with macOS
xar --version

# Basic usage
xar -cf archive.xar file1.txt file2.txt
xar -xf archive.xar

Linux Installation

# Ubuntu/Debian
sudo apt-get install libxar-dev xar

# CentOS/RHEL
sudo yum install libxar-devel xar

# From source
git clone https://github.com/mackyle/xar.git
cd xar/xar
./configure
make
sudo make install

Security Considerations

• Verify archive integrity using built-in checksums
• Be cautious with archives from untrusted sources
• Check file permissions during extraction
• Validate file paths to prevent directory traversal
• Use secure temporary directories for extraction
• Monitor disk space during extraction operations

Best Practices

• Use appropriate compression algorithms for content type
• Include checksums for integrity verification
• Test archives after creation
• Implement proper error handling
• Use temporary directories for safe extraction
• Document archive contents and structure
• Regular integrity checks for long-term storage

Comparison with Other Formats

• Advantages over TAR: Better compression, integrity checking, metadata support
• Advantages over ZIP: Superior compression ratios, XML metadata
• Disadvantages: Less universal support, macOS/Unix focus
• Use cases: Package distribution, system backups, software installers