Protocol Buffers (.proto)

Protocol Buffers (protobuf) is a language-neutral, platform-neutral extensible mechanism for serializing structured data developed by Google. Created by Kenton Varda, Sanjay Ghemawat, and Jeff Dean in 2008, Protocol Buffers provide a way to define data structures in a simple language and generate code for multiple programming languages. It's widely used in microservices, API definitions, and data storage systems.

Technical Overview

Protocol Buffers use a binary serialization format that is smaller, faster, and simpler than XML or JSON. The .proto files contain the schema definitions that describe the structure of data, which are then compiled to generate code in various programming languages. This approach ensures type safety, backward compatibility, and efficient serialization/deserialization.

Key Features

• Language Agnostic: Code generation for 20+ programming languages
• Efficient Serialization: Compact binary format with fast parsing
• Schema Evolution: Backward and forward compatibility support
• Type Safety: Strong typing with validation
• Extensibility: Support for extensions and custom options
• gRPC Integration: Native support for gRPC service definitions

Protocol Buffer Syntax

Basic Message Definition

// proto3 syntax (recommended)
syntax = "proto3";

// Package declaration
package example.v1;

// Import other proto files
import "google/protobuf/timestamp.proto";
import "google/protobuf/duration.proto";

// Message definition
message Person {
  // Fields with unique numbers
  string first_name = 1;
  string last_name = 2;
  int32 age = 3;
  string email = 4;
  repeated string phone_numbers = 5;
  
  // Nested message
  Address address = 6;
  
  // Enum field
  Gender gender = 7;
  
  // Optional field (proto3)
  optional string middle_name = 8;
  
  // Timestamp
  google.protobuf.Timestamp created_at = 9;
}

// Nested message definition
message Address {
  string street = 1;
  string city = 2;
  string state = 3;
  string postal_code = 4;
  string country = 5;
}

// Enum definition
enum Gender {
  GENDER_UNSPECIFIED = 0;  // Default value must be 0
  GENDER_MALE = 1;
  GENDER_FEMALE = 2;
  GENDER_OTHER = 3;
}

Data Types

syntax = "proto3";

message DataTypes {
  // Scalar types
  double price = 1;          // 64-bit floating point
  float rating = 2;          // 32-bit floating point
  int32 count = 3;           // 32-bit signed integer
  int64 large_number = 4;    // 64-bit signed integer
  uint32 positive_count = 5; // 32-bit unsigned integer
  uint64 very_large = 6;     // 64-bit unsigned integer
  sint32 signed_count = 7;   // Signed 32-bit (efficient for negative)
  sint64 signed_large = 8;   // Signed 64-bit (efficient for negative)
  fixed32 fixed_int = 9;     // Fixed 32-bit (4 bytes)
  fixed64 fixed_large = 10;  // Fixed 64-bit (8 bytes)
  sfixed32 signed_fixed = 11; // Signed fixed 32-bit
  sfixed64 signed_fixed64 = 12; // Signed fixed 64-bit
  bool is_active = 13;       // Boolean
  string name = 14;          // UTF-8 string
  bytes data = 15;           // Binary data
  
  // Repeated fields (arrays)
  repeated string tags = 16;
  repeated int32 scores = 17;
  
  // Map fields
  map<string, string> metadata = 18;
  map<int32, Person> users = 19;
}

Advanced Features

syntax = "proto3";

// Options
option java_package = "com.example.tutorial";
option java_outer_classname = "AddressBookProtos";
option csharp_namespace = "Example.Tutorial";
option go_package = "github.com/example/tutorial";

// Message with oneof (union)
message SearchRequest {
  string query = 1;
  int32 page_number = 2;
  int32 result_per_page = 3;
  
  // Only one of these can be set
  oneof filter {
    string category = 4;
    string tag = 5;
    DateRange date_range = 6;
  }
}

message DateRange {
  google.protobuf.Timestamp start = 1;
  google.protobuf.Timestamp end = 2;
}

// Message with reserved fields
message Product {
  reserved 2, 15, 9 to 11;  // Reserved field numbers
  reserved "foo", "bar";     // Reserved field names
  
  string name = 1;
  double price = 3;
  string description = 4;
  repeated string categories = 5;
  
  // Any type for dynamic content
  google.protobuf.Any metadata = 12;
}

// Extensions (proto2 feature)
message ExtendableMessage {
  extensions 100 to 199;
}

extend ExtendableMessage {
  optional string custom_field = 100;
}

// Custom options
import "google/protobuf/descriptor.proto";

extend google.protobuf.FieldOptions {
  string my_field_option = 50001;
}

message MyMessage {
  string field = 1 [(my_field_option) = "custom value"];
}

Service Definitions (gRPC)

Basic Service

syntax = "proto3";

package user.v1;

import "google/protobuf/empty.proto";
import "google/protobuf/timestamp.proto";

// User service definition
service UserService {
  // Unary RPC
  rpc GetUser(GetUserRequest) returns (GetUserResponse);
  
  // Server streaming RPC
  rpc ListUsers(ListUsersRequest) returns (stream User);
  
  // Client streaming RPC
  rpc CreateUsers(stream CreateUserRequest) returns (CreateUsersResponse);
  
  // Bidirectional streaming RPC
  rpc ChatWithUser(stream ChatMessage) returns (stream ChatMessage);
  
  // Empty response
  rpc DeleteUser(DeleteUserRequest) returns (google.protobuf.Empty);
}

message GetUserRequest {
  string user_id = 1;
}

message GetUserResponse {
  User user = 1;
}

message ListUsersRequest {
  int32 page_size = 1;
  string page_token = 2;
  string filter = 3;
}

message CreateUserRequest {
  User user = 1;
}

message CreateUsersResponse {
  repeated User users = 1;
  int32 created_count = 2;
}

message DeleteUserRequest {
  string user_id = 1;
}

message ChatMessage {
  string user_id = 1;
  string content = 2;
  google.protobuf.Timestamp timestamp = 3;
}

message User {
  string id = 1;
  string email = 2;
  string display_name = 3;
  UserStatus status = 4;
  google.protobuf.Timestamp created_at = 5;
  google.protobuf.Timestamp updated_at = 6;
}

enum UserStatus {
  USER_STATUS_UNSPECIFIED = 0;
  USER_STATUS_ACTIVE = 1;
  USER_STATUS_INACTIVE = 2;
  USER_STATUS_SUSPENDED = 3;
}

RESTful API with gRPC-Gateway

syntax = "proto3";

package api.v1;

import "google/api/annotations.proto";
import "google/protobuf/empty.proto";

service BookService {
  rpc ListBooks(ListBooksRequest) returns (ListBooksResponse) {
    option (google.api.http) = {
      get: "/v1/books"
    };
  }
  
  rpc GetBook(GetBookRequest) returns (Book) {
    option (google.api.http) = {
      get: "/v1/books/{book_id}"
    };
  }
  
  rpc CreateBook(CreateBookRequest) returns (Book) {
    option (google.api.http) = {
      post: "/v1/books"
      body: "book"
    };
  }
  
  rpc UpdateBook(UpdateBookRequest) returns (Book) {
    option (google.api.http) = {
      put: "/v1/books/{book.id}"
      body: "book"
    };
  }
  
  rpc DeleteBook(DeleteBookRequest) returns (google.protobuf.Empty) {
    option (google.api.http) = {
      delete: "/v1/books/{book_id}"
    };
  }
}

message Book {
  string id = 1;
  string title = 2;
  string author = 3;
  string isbn = 4;
  double price = 5;
  repeated string genres = 6;
  int32 pages = 7;
  string publisher = 8;
  google.protobuf.Timestamp published_date = 9;
}

message ListBooksRequest {
  int32 page_size = 1;
  string page_token = 2;
  string filter = 3;
  string order_by = 4;
}

message ListBooksResponse {
  repeated Book books = 1;
  string next_page_token = 2;
  int32 total_size = 3;
}

message GetBookRequest {
  string book_id = 1;
}

message CreateBookRequest {
  Book book = 1;
}

message UpdateBookRequest {
  Book book = 1;
  google.protobuf.FieldMask update_mask = 2;
}

message DeleteBookRequest {
  string book_id = 1;
}

Code Generation and Usage

Code Generation

# Install Protocol Compiler
# Ubuntu/Debian
sudo apt install protobuf-compiler

# macOS
brew install protobuf

# Generate code for multiple languages
protoc --proto_path=. \
  --go_out=. --go_opt=paths=source_relative \
  --go-grpc_out=. --go-grpc_opt=paths=source_relative \
  --python_out=. \
  --java_out=. \
  --csharp_out=. \
  user.proto

Go Implementation

// user.pb.go (generated)
package main

import (
    "context"
    "log"
    
    "google.golang.org/grpc"
    pb "example.com/user/v1"
)

// Server implementation
type userServiceServer struct {
    pb.UnimplementedUserServiceServer
    users map[string]*pb.User
}

func (s *userServiceServer) GetUser(ctx context.Context, req *pb.GetUserRequest) (*pb.GetUserResponse, error) {
    user, exists := s.users[req.UserId]
    if !exists {
        return nil, status.Errorf(codes.NotFound, "user not found: %s", req.UserId)
    }
    
    return &pb.GetUserResponse{
        User: user,
    }, nil
}

func (s *userServiceServer) ListUsers(req *pb.ListUsersRequest, stream pb.UserService_ListUsersServer) error {
    for _, user := range s.users {
        if err := stream.Send(user); err != nil {
            return err
        }
    }
    return nil
}

// Client usage
func main() {
    conn, err := grpc.Dial("localhost:50051", grpc.WithInsecure())
    if err != nil {
        log.Fatalf("Failed to connect: %v", err)
    }
    defer conn.Close()
    
    client := pb.NewUserServiceClient(conn)
    
    // Call the service
    resp, err := client.GetUser(context.Background(), &pb.GetUserRequest{
        UserId: "123",
    })
    if err != nil {
        log.Fatalf("GetUser failed: %v", err)
    }
    
    log.Printf("User: %v", resp.User)
}

Python Implementation

# user_pb2.py and user_pb2_grpc.py (generated)
import grpc
from concurrent import futures
import user_pb2
import user_pb2_grpc

class UserServiceServicer(user_pb2_grpc.UserServiceServicer):
    def __init__(self):
        self.users = {}
    
    def GetUser(self, request, context):
        user_id = request.user_id
        if user_id not in self.users:
            context.set_code(grpc.StatusCode.NOT_FOUND)
            context.set_details(f'User not found: {user_id}')
            return user_pb2.GetUserResponse()
        
        return user_pb2.GetUserResponse(user=self.users[user_id])
    
    def ListUsers(self, request, context):
        for user in self.users.values():
            yield user

def serve():
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    user_pb2_grpc.add_UserServiceServicer_to_server(
        UserServiceServicer(), server
    )
    listen_addr = '[::]:50051'
    server.add_insecure_port(listen_addr)
    server.start()
    server.wait_for_termination()

# Client usage
def run_client():
    channel = grpc.insecure_channel('localhost:50051')
    stub = user_pb2_grpc.UserServiceStub(channel)
    
    response = stub.GetUser(user_pb2.GetUserRequest(user_id='123'))
    print(f"User: {response.user}")
    
    # Streaming example
    for user in stub.ListUsers(user_pb2.ListUsersRequest()):
        print(f"User: {user}")

if __name__ == '__main__':
    serve()

JavaScript/TypeScript Implementation

// user_pb.d.ts and user_grpc_pb.d.ts (generated)
import * as grpc from '@grpc/grpc-js';
import { UserServiceClient } from './user_grpc_pb';
import { GetUserRequest, GetUserResponse, User } from './user_pb';

// Client usage
const client = new UserServiceClient(
    'localhost:50051',
    grpc.credentials.createInsecure()
);

// Unary call
const request = new GetUserRequest();
request.setUserId('123');

client.getUser(request, (error, response) => {
    if (error) {
        console.error('Error:', error);
        return;
    }
    
    const user = response.getUser();
    console.log('User:', user?.toObject());
});

// Promise-based wrapper
function getUserAsync(userId: string): Promise<User> {
    return new Promise((resolve, reject) => {
        const request = new GetUserRequest();
        request.setUserId(userId);
        
        client.getUser(request, (error, response) => {
            if (error) {
                reject(error);
                return;
            }
            
            const user = response.getUser();
            if (user) {
                resolve(user);
            } else {
                reject(new Error('User not found'));
            }
        });
    });
}

// Usage with async/await
async function main() {
    try {
        const user = await getUserAsync('123');
        console.log('User name:', user.getDisplayName());
    } catch (error) {
        console.error('Error:', error);
    }
}

Advanced Features and Patterns

Schema Evolution

// Version 1
message UserV1 {
  string name = 1;
  int32 age = 2;
}

// Version 2 - Adding fields (backward compatible)
message UserV2 {
  string name = 1;
  int32 age = 2;
  string email = 3;         // New field
  repeated string tags = 4; // New repeated field
}

// Version 3 - Using oneof for optional fields
message UserV3 {
  string name = 1;
  int32 age = 2;
  string email = 3;
  repeated string tags = 4;
  
  // New optional fields using oneof
  oneof contact_preference {
    string phone = 5;
    string slack_handle = 6;
  }
}

// Best practices for evolution
message EvolvedMessage {
  // Never change field numbers
  string id = 1;
  
  // Add new fields with new numbers
  string name = 2;
  optional string description = 3; // Added in v2
  
  // Use reserved for removed fields
  reserved 4, 5;
  reserved "old_field", "deprecated_field";
  
  // New field
  int64 created_timestamp = 6; // Added in v3
}

Validation and Custom Options

syntax = "proto3";

import "google/protobuf/descriptor.proto";

// Custom validation options
extend google.protobuf.FieldOptions {
  string validation_regex = 50001;
  int32 min_length = 50002;
  int32 max_length = 50003;
  double min_value = 50004;
  double max_value = 50005;
}

message ValidatedUser {
  string email = 1 [(validation_regex) = "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"];
  string username = 2 [(min_length) = 3, (max_length) = 20];
  int32 age = 3 [(min_value) = 0, (max_value) = 150];
  string bio = 4 [(max_length) = 500];
}

// Well-known types usage
import "google/protobuf/timestamp.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/any.proto";

message RichMessage {
  google.protobuf.Timestamp created_at = 1;
  google.protobuf.Duration timeout = 2;
  google.protobuf.Struct metadata = 3;
  repeated google.protobuf.Any attachments = 4;
}

Build Integration

Bazel BUILD file

load("@rules_proto//proto:defs.bzl", "proto_library")
load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")

proto_library(
    name = "user_proto",
    srcs = ["user.proto"],
    deps = [
        "@com_google_protobuf//:timestamp_proto",
        "@com_google_protobuf//:empty_proto",
    ],
)

go_proto_library(
    name = "user_go_proto",
    importpath = "example.com/user/v1",
    proto = ":user_proto",
    deps = [
        "@org_golang_google_protobuf//types/known/timestamppb",
        "@org_golang_google_protobuf//types/known/emptypb",
    ],
)

Makefile Integration

PROTO_DIR := proto
GO_OUT_DIR := pkg/proto
PYTHON_OUT_DIR := python/proto

.PHONY: proto
proto: proto-go proto-python

.PHONY: proto-go
proto-go:
	protoc \
		--proto_path=$(PROTO_DIR) \
		--go_out=$(GO_OUT_DIR) \
		--go_opt=paths=source_relative \
		--go-grpc_out=$(GO_OUT_DIR) \
		--go-grpc_opt=paths=source_relative \
		$(PROTO_DIR)/*.proto

.PHONY: proto-python
proto-python:
	python -m grpc_tools.protoc \
		--proto_path=$(PROTO_DIR) \
		--python_out=$(PYTHON_OUT_DIR) \
		--grpc_python_out=$(PYTHON_OUT_DIR) \
		$(PROTO_DIR)/*.proto

.PHONY: clean
clean:
	rm -rf $(GO_OUT_DIR)/*.pb.go
	rm -rf $(PYTHON_OUT_DIR)/*_pb2.py
	rm -rf $(PYTHON_OUT_DIR)/*_pb2_grpc.py

File Format Details

• MIME Type: text/plain
• File Extension: .proto
• Character Encoding: UTF-8
• Syntax: Protocol Buffer IDL (Interface Definition Language)
• Compiler: protoc (Protocol Compiler)

Protocol Buffers provide a robust foundation for defining data structures and APIs that can evolve over time while maintaining compatibility. Their efficiency, type safety, and cross-language support make them an excellent choice for modern distributed systems and microservices architectures.