从 Python 访问 Apache Ozone

export ARROW_LIBHDFS_DIR=hadoop-3.4.0/lib/native/
export CLASSPATH=$(ozone classpath ozone-tools)

import pyarrow.fs as pafs

# Connect to Ozone using HadoopFileSystem
# "default" tells PyArrow to use the fs.defaultFS property from core-site.xml
fs = pafs.HadoopFileSystem("default")

# Create a directory inside the bucket
fs.create_dir("volume/bucket/aaa")

# Write data to a file
path = "volume/bucket/file1"
with fs.open_output_stream(path) as stream:
  stream.write(b'data')

<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>ofs://om:9862</value>
    <description>Ozone Manager endpoint</description>
  </property>
</configuration>

import boto3

# Create a local file to upload
with open("localfile.txt", "w") as f:
  f.write("Hello from Ozone via Boto3!\n")

# Configure Boto3 client
s3 = boto3.client(
  's3',
  endpoint_url='http://s3g:9878',
  aws_access_key_id='ozone-access-key',
  aws_secret_access_key='ozone-secret-key'
)

# List buckets
response = s3.list_buckets()
print("Buckets:", response['Buckets'])

# Upload the file
s3.upload_file('localfile.txt', 'bucket', 'file.txt')
print("Uploaded 'localfile.txt' to 'bucket/file.txt'")

# Download the file back
s3.download_file('bucket', 'file.txt', 'downloaded.txt')
print("Downloaded 'file.txt' as 'downloaded.txt'")

pip install requests

#!/usr/bin/python
import requests

# Ozone HTTPFS endpoint and file path
host = "http://httpfs:14000"
volume = "vol1"
bucket = "bucket1"
filename = "hello.txt"
path = f"/webhdfs/v1/{volume}/{bucket}/{filename}"
user = "ozone"  # can be any value in simple auth mode

# Step 1: Initiate file creation (responds with 307 redirect)
params_create = {
  "op": "CREATE",
  "overwrite": "true",
  "user.name": user
}

print("Creating file...")
resp_create = requests.put(host + path, params=params_create, allow_redirects=False)

if resp_create.status_code != 307:
  print(f"Unexpected response: {resp_create.status_code}")
  print(resp_create.text)
  exit(1)

redirect_url = resp_create.headers['Location']
print(f"Redirected to: {redirect_url}")

# Step 2: Write data to the redirected location with correct headers
headers = {"Content-Type": "application/octet-stream"}
content = b"Hello from Ozone HTTPFS!\n"

resp_upload = requests.put(redirect_url, data=content, headers=headers)
if resp_upload.status_code != 201:
  print(f"Upload failed: {resp_upload.status_code}")
  print(resp_upload.text)
  exit(1)
print("File created successfully.")

# Step 3: Read the file back
params_open = {
  "op": "OPEN",
  "user.name": user
}

print("Reading file...")
resp_read = requests.get(host + path, params=params_open, allow_redirects=True)
if resp_read.ok:
  print("File contents:")
  print(resp_read.text)
else:
  print(f"Read failed: {resp_read.status_code}")
  print(resp_read.text)

pip install fsspec

from fsspec.implementations.webhdfs import WebHDFS

fs = WebHDFS(host='httpfs', port=14000, user='ozone')

# Read a file from /vol1/bucket1/hello.txt
file_path = "/vol1/bucket1/hello.txt"

with fs.open(file_path, mode='rb') as f:
  content = f.read()
  print("File contents:")
  print(content.decode('utf-8'))