Deploying a fine-tuned model

import os
import time
from together import Together

client = Together(api_key=os.environ.get("TOGETHER_API_KEY"))

# 1. Get the output model name from the completed job
status = client.fine_tuning.retrieve(id="ft-xxxx-yyyy")
output_model = status.x_model_output_name

# 2. Create the endpoint
endpoint = client.endpoints.create(
    display_name="My fine-tuned endpoint",
    model=output_model,
    hardware="4x_nvidia_h100_80gb_sxm",
    autoscaling={"min_replicas": 1, "max_replicas": 1},
)

# 3. Wait until ready, then query
while True:
    ep = client.endpoints.retrieve(endpoint.id)
    if ep.state == "STARTED":
        break
    if ep.state in ("FAILED", "STOPPED"):
        raise RuntimeError(f"Endpoint ended with state: {ep.state}")
    time.sleep(30)

response = client.chat.completions.create(
    model=endpoint.name,
    messages=[
        {
            "role": "user",
            "content": "What are some fun things to do in New York?",
        }
    ],
)
print(response.choices[0].message.content)

# 4. Delete when done to stop billing
client.endpoints.delete(endpoint.id)