From efefbbb8b940a057c6225b1a636c1e387c28e2b5 Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Tue, 25 Nov 2025 13:11:16 +0000 Subject: [PATCH 01/11] add price estimation --- src/together/cli/api/finetune.py | 64 +++++++++--- src/together/resources/finetune.py | 157 ++++++++++++++++++++++++++++- src/together/types/__init__.py | 4 + src/together/types/finetune.py | 26 +++++ 4 files changed, 238 insertions(+), 13 deletions(-) diff --git a/src/together/cli/api/finetune.py b/src/together/cli/api/finetune.py index c14081e..80b3b6b 100644 --- a/src/together/cli/api/finetune.py +++ b/src/together/cli/api/finetune.py @@ -17,6 +17,8 @@ DownloadCheckpointType, FinetuneEventType, FinetuneTrainingLimits, + FullTrainingType, + LoRATrainingType, ) from together.utils import ( finetune_price_to_dollars, @@ -36,6 +38,15 @@ "Do you want to proceed?" ) +_PRICE_ESTIMATION_CONFIRMATION_MESSAGE = ( + "The estimated price of the fine-tuning job is {} which is significantly " + "greater than your current credit limit and balance. " + "It will likely fail due to insufficient funds. " + "Please consider increasing your credit limit at https://api.together.xyz/settings/profile\n" + "You can pass `-y` or `--confirm` to your command to skip this message.\n\n" + "Do you want to proceed?" +) + class DownloadCheckpointTypeChoice(click.Choice): def __init__(self) -> None: @@ -358,20 +369,49 @@ def create( ) if confirm or click.confirm(_CONFIRMATION_MESSAGE, default=True, show_default=True): - response = client.fine_tuning.create( - **training_args, - verbose=True, + price_estimation_response = client.fine_tuning.estimate_price( + training_file=training_file, + validation_file=validation_file, + model=model, + n_epochs=n_epochs, + n_evals=n_evals, + training_type="lora" if lora else "full", + training_method=training_method, ) - - report_string = f"Successfully submitted a fine-tuning job {response.id}" - if response.created_at is not None: - created_time = datetime.strptime( - response.created_at, "%Y-%m-%dT%H:%M:%S.%f%z" + proceed = ( + confirm + or price_estimation_response.allowed_to_proceed + or ( + not price_estimation_response.allowed_to_proceed + and click.confirm( + click.style( + _PRICE_ESTIMATION_CONFIRMATION_MESSAGE.format( + price_estimation_response.estimated_total_price + ), + fg="red", + bold=True, + ), + default=True, + show_default=True, + ) + ) + ) + if proceed: + response = client.fine_tuning.create( + **training_args, + verbose=True, ) - # created_at reports UTC time, we use .astimezone() to convert to local time - formatted_time = created_time.astimezone().strftime("%m/%d/%Y, %H:%M:%S") - report_string += f" at {formatted_time}" - rprint(report_string) + report_string = f"Successfully submitted a fine-tuning job {response.id}" + if response.created_at is not None: + created_time = datetime.strptime( + response.created_at, "%Y-%m-%dT%H:%M:%S.%f%z" + ) + # created_at reports UTC time, we use .astimezone() to convert to local time + formatted_time = created_time.astimezone().strftime( + "%m/%d/%Y, %H:%M:%S" + ) + report_string += f" at {formatted_time}" + rprint(report_string) else: click.echo("No confirmation received, stopping job launch") diff --git a/src/together/resources/finetune.py b/src/together/resources/finetune.py index 2b3a652..adb52dc 100644 --- a/src/together/resources/finetune.py +++ b/src/together/resources/finetune.py @@ -20,6 +20,8 @@ FinetuneLRScheduler, FinetuneRequest, FinetuneResponse, + FinetunePriceEstimationRequest, + FinetunePriceEstimationResponse, FinetuneTrainingLimits, FullTrainingType, LinearLRScheduler, @@ -31,7 +33,7 @@ TrainingMethodSFT, TrainingType, ) -from together.types.finetune import DownloadCheckpointType +from together.types.finetune import DownloadCheckpointType, TrainingMethod from together.utils import log_warn_once, normalize_key @@ -42,6 +44,12 @@ TrainingMethodSFT().method, TrainingMethodDPO().method, } +_CONFIRMATION_MESSAGE_INSUFFICIENT_FUNDS = ( + "The estimated price of the fine-tuning job is {} which is significantly " + "greater than your current credit limit and balance. " + "It will likely fail due to insufficient funds. " + "Please proceed at your own risk." +) def create_finetune_request( @@ -474,11 +482,29 @@ def create( hf_output_repo_name=hf_output_repo_name, ) + price_estimation_result = self.estimate_price( + training_file=training_file, + validation_file=validation_file, + model=model_name, + n_epochs=n_epochs, + n_evals=n_evals, + training_type="lora" if lora else "full", + training_method=training_method, + ) + if verbose: rprint( "Submitting a fine-tuning job with the following parameters:", finetune_request, ) + if not price_estimation_result.allowed_to_proceed: + rprint( + "[red]" + + _CONFIRMATION_MESSAGE_INSUFFICIENT_FUNDS.format( + price_estimation_result.estimated_total_price + ) + + "[/red]", + ) parameter_payload = finetune_request.model_dump(exclude_none=True) response, _, _ = requestor.request( @@ -493,6 +519,73 @@ def create( return FinetuneResponse(**response.data) + def estimate_price( + self, + *, + training_file: str, + model: str | None, + validation_file: str | None = None, + n_epochs: int | None = None, + n_evals: int | None = None, + training_type: str = "lora", + training_method: str = "sft", + ) -> FinetunePriceEstimationResponse: + """ + Estimates the price of a fine-tuning job + + Args: + request (FinetunePriceEstimationRequest): Request object containing the parameters for the price estimation. + + Returns: + FinetunePriceEstimationResponse: Object containing the estimated price. + """ + training_type_cls: TrainingType | None = None + training_method_cls: TrainingMethod | None = None + + if training_method == "sft": + training_method_cls = TrainingMethodSFT(method="sft") + elif training_method == "dpo": + training_method_cls = TrainingMethodDPO(method="dpo") + else: + raise ValueError(f"Unknown training method: {training_method}") + + if training_type.lower() == "lora": + training_type_cls = LoRATrainingType( + type="Lora", + lora_r=16, + lora_alpha=16, + lora_dropout=0.0, + lora_trainable_modules="all-linear", + ) + elif training_type.lower() == "full": + training_type_cls = FullTrainingType(type="Full") + else: + raise ValueError(f"Unknown training type: {training_type}") + + request = FinetunePriceEstimationRequest( + training_file=training_file, + validation_file=validation_file, + model=model, + n_epochs=n_epochs, + n_evals=n_evals, + training_type=training_type_cls, + training_method=training_method_cls, + ) + parameter_payload = request.model_dump(exclude_none=True) + requestor = api_requestor.APIRequestor( + client=self._client, + ) + + response, _, _ = requestor.request( + options=TogetherRequest( + method="POST", url="fine-tunes/estimate-price", params=parameter_payload + ), + stream=False, + ) + assert isinstance(response, TogetherResponse) + + return FinetunePriceEstimationResponse(**response.data) + def list(self) -> FinetuneList: """ Lists fine-tune job history @@ -941,11 +1034,29 @@ async def create( hf_output_repo_name=hf_output_repo_name, ) + price_estimation_result = await self.estimate_price( + training_file=training_file, + validation_file=validation_file, + model=model_name, + n_epochs=n_epochs, + n_evals=n_evals, + training_type=finetune_request.training_type, + training_method=finetune_request.training_method, + ) + if verbose: rprint( "Submitting a fine-tuning job with the following parameters:", finetune_request, ) + if not price_estimation_result.allowed_to_proceed: + rprint( + "[red]" + + _CONFIRMATION_MESSAGE_INSUFFICIENT_FUNDS.format( + price_estimation_result.estimated_total_price + ) + + "[/red]", + ) parameter_payload = finetune_request.model_dump(exclude_none=True) response, _, _ = await requestor.arequest( @@ -961,6 +1072,50 @@ async def create( return FinetuneResponse(**response.data) + async def estimate_price( + self, + *, + training_file: str, + model: str, + validation_file: str | None = None, + n_epochs: int | None = None, + n_evals: int | None = None, + training_type: TrainingType | None = None, + training_method: TrainingMethodSFT | TrainingMethodDPO | None = None, + ) -> FinetunePriceEstimationResponse: + """ + Async method to estimate the price of a fine-tuning job + + Args: + request (FinetunePriceEstimationRequest): Request object containing the parameters for the price estimation. + + Returns: + FinetunePriceEstimationResponse: Object containing the estimated price. + """ + request = FinetunePriceEstimationRequest( + training_file=training_file, + validation_file=validation_file, + model=model, + n_epochs=n_epochs, + n_evals=n_evals, + training_type=training_type, + training_method=training_method, + ) + parameter_payload = request.model_dump(exclude_none=True) + requestor = api_requestor.APIRequestor( + client=self._client, + ) + + response, _, _ = await requestor.arequest( + options=TogetherRequest( + method="POST", url="fine-tunes/estimate-price", params=parameter_payload + ), + stream=False, + ) + assert isinstance(response, TogetherResponse) + + return FinetunePriceEstimationResponse(**response.data) + async def list(self) -> FinetuneList: """ Async method to list fine-tune job history diff --git a/src/together/types/__init__.py b/src/together/types/__init__.py index f4dd737..61c054a 100644 --- a/src/together/types/__init__.py +++ b/src/together/types/__init__.py @@ -54,6 +54,8 @@ FinetuneListEvents, FinetuneRequest, FinetuneResponse, + FinetunePriceEstimationRequest, + FinetunePriceEstimationResponse, FinetuneDeleteResponse, FinetuneTrainingLimits, FullTrainingType, @@ -103,6 +105,8 @@ "FinetuneDeleteResponse", "FinetuneDownloadResult", "FinetuneLRScheduler", + "FinetunePriceEstimationRequest", + "FinetunePriceEstimationResponse", "LinearLRScheduler", "LinearLRSchedulerArgs", "CosineLRScheduler", diff --git a/src/together/types/finetune.py b/src/together/types/finetune.py index 52c802b..4ebb521 100644 --- a/src/together/types/finetune.py +++ b/src/together/types/finetune.py @@ -308,6 +308,32 @@ def validate_training_type(cls, v: TrainingType) -> TrainingType: raise ValueError("Unknown training type") +class FinetunePriceEstimationRequest(BaseModel): + """ + Fine-tune price estimation request type + """ + + training_file: str + validation_file: str | None = None + model: str + n_epochs: int | None = None + n_evals: int | None = None + training_type: TrainingType | None = None + training_method: TrainingMethodSFT | TrainingMethodDPO + + +class FinetunePriceEstimationResponse(BaseModel): + """ + Fine-tune price estimation response type + """ + + estimated_total_price: float + user_limit: float + estimated_train_token_count: int + estimated_eval_token_count: int + allowed_to_proceed: bool + + class FinetuneList(BaseModel): # object type object: Literal["list"] | None = None From 2ee9bbcd3b9a92055ec28f0008e0e10bdfb807e4 Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Tue, 25 Nov 2025 16:13:48 +0000 Subject: [PATCH 02/11] comments from the review --- src/together/cli/api/finetune.py | 2 ++ src/together/resources/finetune.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/together/cli/api/finetune.py b/src/together/cli/api/finetune.py index 80b3b6b..cc35def 100644 --- a/src/together/cli/api/finetune.py +++ b/src/together/cli/api/finetune.py @@ -412,6 +412,8 @@ def create( ) report_string += f" at {formatted_time}" rprint(report_string) + else: + click.echo("No confirmation received, stopping job launch") else: click.echo("No confirmation received, stopping job launch") diff --git a/src/together/resources/finetune.py b/src/together/resources/finetune.py index adb52dc..6a84da3 100644 --- a/src/together/resources/finetune.py +++ b/src/together/resources/finetune.py @@ -550,6 +550,8 @@ def estimate_price( raise ValueError(f"Unknown training method: {training_method}") if training_type.lower() == "lora": + # parameters of lora are unused in price estimation + # but we need to set them to valid values training_type_cls = LoRATrainingType( type="Lora", lora_r=16, From d35e7705943680197296e55b3e3d05d5668837d8 Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Wed, 26 Nov 2025 14:04:15 +0000 Subject: [PATCH 03/11] =?UTF-8?q?=D0=B0=D0=B4=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/together/cli/api/finetune.py | 94 ++++++++++++++---------------- src/together/resources/finetune.py | 6 +- 2 files changed, 47 insertions(+), 53 deletions(-) diff --git a/src/together/cli/api/finetune.py b/src/together/cli/api/finetune.py index cc35def..cf8e8b1 100644 --- a/src/together/cli/api/finetune.py +++ b/src/together/cli/api/finetune.py @@ -31,20 +31,19 @@ _CONFIRMATION_MESSAGE = ( "You are about to create a fine-tuning job. " - "The cost of your job will be determined by the model size, the number of tokens " + "The estimated price of this job is {price}. " + "The actual cost of your job will be determined by the model size, the number of tokens " "in the training file, the number of tokens in the validation file, the number of epochs, and " - "the number of evaluations. Visit https://www.together.ai/pricing to get a price estimate.\n" + "the number of evaluations. Visit https://www.together.ai/pricing to learn more about pricing.\n" + "{warning}" "You can pass `-y` or `--confirm` to your command to skip this message.\n\n" "Do you want to proceed?" ) -_PRICE_ESTIMATION_CONFIRMATION_MESSAGE = ( - "The estimated price of the fine-tuning job is {} which is significantly " - "greater than your current credit limit and balance. " +_WARNING_MESSAGE_INSUFFICIENT_FUNDS = ( + "The estimated price of this job is significantly greater than your current credit limit and balance. " "It will likely fail due to insufficient funds. " "Please consider increasing your credit limit at https://api.together.xyz/settings/profile\n" - "You can pass `-y` or `--confirm` to your command to skip this message.\n\n" - "Do you want to proceed?" ) @@ -368,52 +367,47 @@ def create( "You have specified a number of evaluation loops but no validation file." ) - if confirm or click.confirm(_CONFIRMATION_MESSAGE, default=True, show_default=True): - price_estimation_response = client.fine_tuning.estimate_price( - training_file=training_file, - validation_file=validation_file, - model=model, - n_epochs=n_epochs, - n_evals=n_evals, - training_type="lora" if lora else "full", - training_method=training_method, + finetune_price_estimation_result = client.fine_tuning.estimate_price( + training_file=training_file, + validation_file=validation_file, + model=model, + n_epochs=n_epochs, + n_evals=n_evals, + training_type="lora" if lora else "full", + training_method=training_method, + ) + + price = click.style( + f"${finetune_price_estimation_result.estimated_total_price:.2f}", + bold=True, + ) + + if not finetune_price_estimation_result.allowed_to_proceed: + warning = click.style(_WARNING_MESSAGE_INSUFFICIENT_FUNDS, fg="red", bold=True) + else: + warning = "" + + confirmation_message = _CONFIRMATION_MESSAGE.format( + price=price, + warning=warning, + ) + + if confirm or click.confirm(confirmation_message, default=True, show_default=True): + response = client.fine_tuning.create( + **training_args, + verbose=True, ) - proceed = ( - confirm - or price_estimation_response.allowed_to_proceed - or ( - not price_estimation_response.allowed_to_proceed - and click.confirm( - click.style( - _PRICE_ESTIMATION_CONFIRMATION_MESSAGE.format( - price_estimation_response.estimated_total_price - ), - fg="red", - bold=True, - ), - default=True, - show_default=True, - ) + report_string = f"Successfully submitted a fine-tuning job {response.id}" + if response.created_at is not None: + created_time = datetime.strptime( + response.created_at, "%Y-%m-%dT%H:%M:%S.%f%z" ) - ) - if proceed: - response = client.fine_tuning.create( - **training_args, - verbose=True, + # created_at reports UTC time, we use .astimezone() to convert to local time + formatted_time = created_time.astimezone().strftime( + "%m/%d/%Y, %H:%M:%S" ) - report_string = f"Successfully submitted a fine-tuning job {response.id}" - if response.created_at is not None: - created_time = datetime.strptime( - response.created_at, "%Y-%m-%dT%H:%M:%S.%f%z" - ) - # created_at reports UTC time, we use .astimezone() to convert to local time - formatted_time = created_time.astimezone().strftime( - "%m/%d/%Y, %H:%M:%S" - ) - report_string += f" at {formatted_time}" - rprint(report_string) - else: - click.echo("No confirmation received, stopping job launch") + report_string += f" at {formatted_time}" + rprint(report_string) else: click.echo("No confirmation received, stopping job launch") diff --git a/src/together/resources/finetune.py b/src/together/resources/finetune.py index 6a84da3..afe1452 100644 --- a/src/together/resources/finetune.py +++ b/src/together/resources/finetune.py @@ -44,7 +44,7 @@ TrainingMethodSFT().method, TrainingMethodDPO().method, } -_CONFIRMATION_MESSAGE_INSUFFICIENT_FUNDS = ( +_WARNING_MESSAGE_INSUFFICIENT_FUNDS = ( "The estimated price of the fine-tuning job is {} which is significantly " "greater than your current credit limit and balance. " "It will likely fail due to insufficient funds. " @@ -500,7 +500,7 @@ def create( if not price_estimation_result.allowed_to_proceed: rprint( "[red]" - + _CONFIRMATION_MESSAGE_INSUFFICIENT_FUNDS.format( + + _WARNING_MESSAGE_INSUFFICIENT_FUNDS.format( price_estimation_result.estimated_total_price ) + "[/red]", @@ -1054,7 +1054,7 @@ async def create( if not price_estimation_result.allowed_to_proceed: rprint( "[red]" - + _CONFIRMATION_MESSAGE_INSUFFICIENT_FUNDS.format( + + _WARNING_MESSAGE_INSUFFICIENT_FUNDS.format( price_estimation_result.estimated_total_price ) + "[/red]", From 5eaea008fa720dadd3bf3d57b62fa963aad0d9a2 Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Wed, 26 Nov 2025 14:05:51 +0000 Subject: [PATCH 04/11] code style --- src/together/cli/api/finetune.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/together/cli/api/finetune.py b/src/together/cli/api/finetune.py index cf8e8b1..ca7f2f0 100644 --- a/src/together/cli/api/finetune.py +++ b/src/together/cli/api/finetune.py @@ -376,17 +376,17 @@ def create( training_type="lora" if lora else "full", training_method=training_method, ) - + price = click.style( f"${finetune_price_estimation_result.estimated_total_price:.2f}", bold=True, ) - + if not finetune_price_estimation_result.allowed_to_proceed: warning = click.style(_WARNING_MESSAGE_INSUFFICIENT_FUNDS, fg="red", bold=True) else: warning = "" - + confirmation_message = _CONFIRMATION_MESSAGE.format( price=price, warning=warning, @@ -403,9 +403,7 @@ def create( response.created_at, "%Y-%m-%dT%H:%M:%S.%f%z" ) # created_at reports UTC time, we use .astimezone() to convert to local time - formatted_time = created_time.astimezone().strftime( - "%m/%d/%Y, %H:%M:%S" - ) + formatted_time = created_time.astimezone().strftime("%m/%d/%Y, %H:%M:%S") report_string += f" at {formatted_time}" rprint(report_string) else: From f73533d01dc4616ebd70090fd7576bb2c3a35b48 Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Wed, 26 Nov 2025 14:14:59 +0000 Subject: [PATCH 05/11] add combined --- src/together/cli/api/finetune.py | 2 +- src/together/resources/finetune.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/together/cli/api/finetune.py b/src/together/cli/api/finetune.py index ca7f2f0..dde1338 100644 --- a/src/together/cli/api/finetune.py +++ b/src/together/cli/api/finetune.py @@ -41,7 +41,7 @@ ) _WARNING_MESSAGE_INSUFFICIENT_FUNDS = ( - "The estimated price of this job is significantly greater than your current credit limit and balance. " + "The estimated price of this job is significantly greater than your current credit limit and balance combined. " "It will likely fail due to insufficient funds. " "Please consider increasing your credit limit at https://api.together.xyz/settings/profile\n" ) diff --git a/src/together/resources/finetune.py b/src/together/resources/finetune.py index afe1452..74bf713 100644 --- a/src/together/resources/finetune.py +++ b/src/together/resources/finetune.py @@ -46,7 +46,7 @@ } _WARNING_MESSAGE_INSUFFICIENT_FUNDS = ( "The estimated price of the fine-tuning job is {} which is significantly " - "greater than your current credit limit and balance. " + "greater than your current credit limit and balance combined. " "It will likely fail due to insufficient funds. " "Please proceed at your own risk." ) From dea7fdfbf6a91367cf7037903cad5ad0a4da2391 Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Wed, 26 Nov 2025 17:29:12 +0000 Subject: [PATCH 06/11] address None and comments from review --- src/together/cli/api/finetune.py | 4 +- src/together/resources/finetune.py | 110 ++++++++++++++++------- src/together/types/finetune.py | 8 +- tests/unit/test_finetune_resources.py | 123 ++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 39 deletions(-) diff --git a/src/together/cli/api/finetune.py b/src/together/cli/api/finetune.py index dde1338..f84af6f 100644 --- a/src/together/cli/api/finetune.py +++ b/src/together/cli/api/finetune.py @@ -42,8 +42,8 @@ _WARNING_MESSAGE_INSUFFICIENT_FUNDS = ( "The estimated price of this job is significantly greater than your current credit limit and balance combined. " - "It will likely fail due to insufficient funds. " - "Please consider increasing your credit limit at https://api.together.xyz/settings/profile\n" + "It will likely get cancelled due to insufficient funds. " + "Consider increasing your credit limit at https://api.together.xyz/settings/profile\n" ) diff --git a/src/together/resources/finetune.py b/src/together/resources/finetune.py index 74bf713..0483cb2 100644 --- a/src/together/resources/finetune.py +++ b/src/together/resources/finetune.py @@ -47,8 +47,8 @@ _WARNING_MESSAGE_INSUFFICIENT_FUNDS = ( "The estimated price of the fine-tuning job is {} which is significantly " "greater than your current credit limit and balance combined. " - "It will likely fail due to insufficient funds. " - "Please proceed at your own risk." + "It will likely get cancelled due to insufficient funds. " + "Proceed at your own risk." ) @@ -481,16 +481,25 @@ def create( hf_api_token=hf_api_token, hf_output_repo_name=hf_output_repo_name, ) - - price_estimation_result = self.estimate_price( - training_file=training_file, - validation_file=validation_file, - model=model_name, - n_epochs=n_epochs, - n_evals=n_evals, - training_type="lora" if lora else "full", - training_method=training_method, - ) + if from_checkpoint is None: + price_estimation_result = self.estimate_price( + training_file=training_file, + validation_file=validation_file, + model=model_name, + n_epochs=finetune_request.n_epochs, + n_evals=finetune_request.n_evals, + training_type="lora" if lora else "full", + training_method=training_method, + ) + else: + # unsupported case + price_estimation_result = FinetunePriceEstimationResponse( + estimated_total_price=0.0, + allowed_to_proceed=True, + estimated_train_token_count=0, + estimated_eval_token_count=0, + user_limit=0.0, + ) if verbose: rprint( @@ -523,10 +532,10 @@ def estimate_price( self, *, training_file: str, - model: str | None, + model: str, validation_file: str | None = None, - n_epochs: int | None = None, - n_evals: int | None = None, + n_epochs: int | None = 1, + n_evals: int | None = 0, training_type: str = "lora", training_method: str = "sft", ) -> FinetunePriceEstimationResponse: @@ -539,8 +548,8 @@ def estimate_price( Returns: FinetunePriceEstimationResponse: Object containing the estimated price. """ - training_type_cls: TrainingType | None = None - training_method_cls: TrainingMethod | None = None + training_type_cls: TrainingType + training_method_cls: TrainingMethod if training_method == "sft": training_method_cls = TrainingMethodSFT(method="sft") @@ -1036,15 +1045,25 @@ async def create( hf_output_repo_name=hf_output_repo_name, ) - price_estimation_result = await self.estimate_price( - training_file=training_file, - validation_file=validation_file, - model=model_name, - n_epochs=n_epochs, - n_evals=n_evals, - training_type=finetune_request.training_type, - training_method=finetune_request.training_method, - ) + if from_checkpoint is not None: + price_estimation_result = await self.estimate_price( + training_file=training_file, + validation_file=validation_file, + model=model_name, + n_epochs=finetune_request.n_epochs, + n_evals=finetune_request.n_evals, + training_type="lora" if lora else "full", + training_method=training_method, + ) + else: + # unsupported case + price_estimation_result = FinetunePriceEstimationResponse( + estimated_total_price=0.0, + allowed_to_proceed=True, + estimated_train_token_count=0, + estimated_eval_token_count=0, + user_limit=0.0, + ) if verbose: rprint( @@ -1080,13 +1099,13 @@ async def estimate_price( training_file: str, model: str, validation_file: str | None = None, - n_epochs: int | None = None, - n_evals: int | None = None, - training_type: TrainingType | None = None, - training_method: TrainingMethodSFT | TrainingMethodDPO | None = None, + n_epochs: int | None = 1, + n_evals: int | None = 0, + training_type: str = "lora", + training_method: str = "sft", ) -> FinetunePriceEstimationResponse: """ - Async method to estimate the price of a fine-tuning job + Estimates the price of a fine-tuning job Args: request (FinetunePriceEstimationRequest): Request object containing the parameters for the price estimation. @@ -1094,14 +1113,39 @@ async def estimate_price( Returns: FinetunePriceEstimationResponse: Object containing the estimated price. """ + training_type_cls: TrainingType + training_method_cls: TrainingMethod + + if training_method == "sft": + training_method_cls = TrainingMethodSFT(method="sft") + elif training_method == "dpo": + training_method_cls = TrainingMethodDPO(method="dpo") + else: + raise ValueError(f"Unknown training method: {training_method}") + + if training_type.lower() == "lora": + # parameters of lora are unused in price estimation + # but we need to set them to valid values + training_type_cls = LoRATrainingType( + type="Lora", + lora_r=16, + lora_alpha=16, + lora_dropout=0.0, + lora_trainable_modules="all-linear", + ) + elif training_type.lower() == "full": + training_type_cls = FullTrainingType(type="Full") + else: + raise ValueError(f"Unknown training type: {training_type}") + request = FinetunePriceEstimationRequest( training_file=training_file, validation_file=validation_file, model=model, n_epochs=n_epochs, n_evals=n_evals, - training_type=training_type, - training_method=training_method, + training_type=training_type_cls, + training_method=training_method_cls, ) parameter_payload = request.model_dump(exclude_none=True) requestor = api_requestor.APIRequestor( diff --git a/src/together/types/finetune.py b/src/together/types/finetune.py index 4ebb521..286932e 100644 --- a/src/together/types/finetune.py +++ b/src/together/types/finetune.py @@ -316,10 +316,10 @@ class FinetunePriceEstimationRequest(BaseModel): training_file: str validation_file: str | None = None model: str - n_epochs: int | None = None - n_evals: int | None = None - training_type: TrainingType | None = None - training_method: TrainingMethodSFT | TrainingMethodDPO + n_epochs: int + n_evals: int + training_type: TrainingType + training_method: TrainingMethod class FinetunePriceEstimationResponse(BaseModel): diff --git a/tests/unit/test_finetune_resources.py b/tests/unit/test_finetune_resources.py index b72e5b1..3659424 100644 --- a/tests/unit/test_finetune_resources.py +++ b/tests/unit/test_finetune_resources.py @@ -1,6 +1,10 @@ import pytest +from unittest.mock import MagicMock, Mock, patch +from together.client import Together from together.resources.finetune import create_finetune_request +from together.together_response import TogetherResponse +from together.types import TogetherRequest from together.types.finetune import ( FinetuneFullTrainingLimits, FinetuneLoraTrainingLimits, @@ -31,6 +35,41 @@ ) +def mock_request(options: TogetherRequest, *args, **kwargs): + if options.url == "fine-tunes/estimate-price": + return ( + TogetherResponse( + data={ + "estimated_total_price": 100, + "allowed_to_proceed": True, + "estimated_train_token_count": 1000, + "estimated_eval_token_count": 100, + "user_limit": 1000, + }, + headers={}, + ), + None, + None, + ) + elif options.url == "fine-tunes": + return ( + TogetherResponse( + data={ + "id": "ft-12345678-1234-1234-1234-1234567890ab", + }, + headers={}, + ), + None, + None, + ) + else: + return ( + TogetherResponse(data=_MODEL_LIMITS.model_dump(), headers={}), + None, + None, + ) + + def test_simple_request(): request = create_finetune_request( model_limits=_MODEL_LIMITS, @@ -335,3 +374,87 @@ def test_train_on_inputs_not_supported_for_dpo(): training_method="dpo", train_on_inputs=True, ) + + +@patch("together.abstract.api_requestor.APIRequestor.request") +def test_price_estimation_request(mocker): + test_data = [ + { + "training_type": "lora", + "training_method": "sft", + }, + { + "training_type": "lora", + "training_method": "dpo", + }, + { + "training_type": "full", + "training_method": "sft", + }, + ] + mocker.return_value = ( + TogetherResponse( + data={ + "estimated_total_price": 100, + "allowed_to_proceed": True, + "estimated_train_token_count": 1000, + "estimated_eval_token_count": 100, + "user_limit": 1000, + }, + headers={}, + ), + None, + None, + ) + client = Together() + for test_case in test_data: + response = client.fine_tuning.estimate_price( + training_file=_TRAINING_FILE, + model=_MODEL_NAME, + validation_file=_VALIDATION_FILE, + n_epochs=1, + n_evals=0, + training_type=test_case["training_type"], + training_method=test_case["training_method"], + ) + assert response.estimated_total_price > 0 + assert response.allowed_to_proceed + assert response.estimated_train_token_count > 0 + assert response.estimated_eval_token_count > 0 + + +def test_create_ft_job(mocker): + mock_requestor = Mock() + mock_requestor.request = MagicMock() + mock_requestor.request.side_effect = mock_request + mocker.patch( + "together.abstract.api_requestor.APIRequestor", return_value=mock_requestor + ) + + client = Together() + response = client.fine_tuning.create( + training_file=_TRAINING_FILE, + model=_MODEL_NAME, + validation_file=_VALIDATION_FILE, + n_epochs=1, + n_evals=0, + lora=True, + training_method="sft", + ) + + assert mock_requestor.request.call_count == 3 + assert response.id == "ft-12345678-1234-1234-1234-1234567890ab" + + response = client.fine_tuning.create( + training_file=_TRAINING_FILE, + model=None, + validation_file=_VALIDATION_FILE, + n_epochs=1, + n_evals=0, + lora=True, + training_method="sft", + from_checkpoint=_FROM_CHECKPOINT, + ) + + assert mock_requestor.request.call_count == 5 + assert response.id == "ft-12345678-1234-1234-1234-1234567890ab" From a422b53415b4fbcf709a3bd8c72d5042ba83147a Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Wed, 26 Nov 2025 17:31:22 +0000 Subject: [PATCH 07/11] Update test_finetune_resources.py --- tests/unit/test_finetune_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_finetune_resources.py b/tests/unit/test_finetune_resources.py index 3659424..eaba427 100644 --- a/tests/unit/test_finetune_resources.py +++ b/tests/unit/test_finetune_resources.py @@ -431,7 +431,7 @@ def test_create_ft_job(mocker): "together.abstract.api_requestor.APIRequestor", return_value=mock_requestor ) - client = Together() + client = Together(api_key="fake_api_key") response = client.fine_tuning.create( training_file=_TRAINING_FILE, model=_MODEL_NAME, From ca423821f138d4d632556b28d24f9377612d23fd Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Wed, 26 Nov 2025 17:32:21 +0000 Subject: [PATCH 08/11] Update test_finetune_resources.py --- tests/unit/test_finetune_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_finetune_resources.py b/tests/unit/test_finetune_resources.py index eaba427..b48967c 100644 --- a/tests/unit/test_finetune_resources.py +++ b/tests/unit/test_finetune_resources.py @@ -406,7 +406,7 @@ def test_price_estimation_request(mocker): None, None, ) - client = Together() + client = Together(api_key="fake_api_key") for test_case in test_data: response = client.fine_tuning.estimate_price( training_file=_TRAINING_FILE, From 3c11c5f1a9a2f97617ae21f3e5e1b5d350c7431b Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Mon, 1 Dec 2025 09:18:21 -0800 Subject: [PATCH 09/11] fix from_checkpoint and hf_model --- src/together/resources/finetune.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/together/resources/finetune.py b/src/together/resources/finetune.py index 0483cb2..a8932cc 100644 --- a/src/together/resources/finetune.py +++ b/src/together/resources/finetune.py @@ -481,7 +481,7 @@ def create( hf_api_token=hf_api_token, hf_output_repo_name=hf_output_repo_name, ) - if from_checkpoint is None: + if from_checkpoint is None and from_hf_model is None: price_estimation_result = self.estimate_price( training_file=training_file, validation_file=validation_file, @@ -1045,7 +1045,7 @@ async def create( hf_output_repo_name=hf_output_repo_name, ) - if from_checkpoint is not None: + if from_checkpoint is None and from_hf_model is None: price_estimation_result = await self.estimate_price( training_file=training_file, validation_file=validation_file, From dc923a40ee474d52f944faada62b5a222f2dbb55 Mon Sep 17 00:00:00 2001 From: newokaerinasai Date: Wed, 3 Dec 2025 16:05:08 -0800 Subject: [PATCH 10/11] comments from code review --- src/together/resources/finetune.py | 42 ++++++++++++++------------- tests/unit/test_finetune_resources.py | 32 +++++++++----------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/src/together/resources/finetune.py b/src/together/resources/finetune.py index a8932cc..7cd1eb0 100644 --- a/src/together/resources/finetune.py +++ b/src/together/resources/finetune.py @@ -491,22 +491,17 @@ def create( training_type="lora" if lora else "full", training_method=training_method, ) + price_limit_passed = price_estimation_result.allowed_to_proceed else: # unsupported case - price_estimation_result = FinetunePriceEstimationResponse( - estimated_total_price=0.0, - allowed_to_proceed=True, - estimated_train_token_count=0, - estimated_eval_token_count=0, - user_limit=0.0, - ) + price_limit_passed = True if verbose: rprint( "Submitting a fine-tuning job with the following parameters:", finetune_request, ) - if not price_estimation_result.allowed_to_proceed: + if not price_limit_passed: rprint( "[red]" + _WARNING_MESSAGE_INSUFFICIENT_FUNDS.format( @@ -543,10 +538,16 @@ def estimate_price( Estimates the price of a fine-tuning job Args: - request (FinetunePriceEstimationRequest): Request object containing the parameters for the price estimation. + training_file (str): File-ID of a file uploaded to the Together API + model (str): Name of the base model to run fine-tune job on + validation_file (str, optional): File ID of a file uploaded to the Together API for validation. + n_epochs (int, optional): Number of epochs for fine-tuning. Defaults to 1. + n_evals (int, optional): Number of evaluation loops to run. Defaults to 0. + training_type (str, optional): Training type. Defaults to "lora". + training_method (str, optional): Training method. Defaults to "sft". Returns: - FinetunePriceEstimationResponse: Object containing the estimated price. + FinetunePriceEstimationResponse: Object containing the price estimation result. """ training_type_cls: TrainingType training_method_cls: TrainingMethod @@ -1055,22 +1056,17 @@ async def create( training_type="lora" if lora else "full", training_method=training_method, ) + price_limit_passed = price_estimation_result.allowed_to_proceed else: # unsupported case - price_estimation_result = FinetunePriceEstimationResponse( - estimated_total_price=0.0, - allowed_to_proceed=True, - estimated_train_token_count=0, - estimated_eval_token_count=0, - user_limit=0.0, - ) + price_limit_passed = True if verbose: rprint( "Submitting a fine-tuning job with the following parameters:", finetune_request, ) - if not price_estimation_result.allowed_to_proceed: + if not price_limit_passed: rprint( "[red]" + _WARNING_MESSAGE_INSUFFICIENT_FUNDS.format( @@ -1108,10 +1104,16 @@ async def estimate_price( Estimates the price of a fine-tuning job Args: - request (FinetunePriceEstimationRequest): Request object containing the parameters for the price estimation. + training_file (str): File-ID of a file uploaded to the Together API + model (str): Name of the base model to run fine-tune job on + validation_file (str, optional): File ID of a file uploaded to the Together API for validation. + n_epochs (int, optional): Number of epochs for fine-tuning. Defaults to 1. + n_evals (int, optional): Number of evaluation loops to run. Defaults to 0. + training_type (str, optional): Training type. Defaults to "lora". + training_method (str, optional): Training method. Defaults to "sft". Returns: - FinetunePriceEstimationResponse: Object containing the estimated price. + FinetunePriceEstimationResponse: Object containing the price estimation result. """ training_type_cls: TrainingType training_method_cls: TrainingMethod diff --git a/tests/unit/test_finetune_resources.py b/tests/unit/test_finetune_resources.py index b48967c..6020a0c 100644 --- a/tests/unit/test_finetune_resources.py +++ b/tests/unit/test_finetune_resources.py @@ -16,6 +16,7 @@ _TRAINING_FILE = "file-7dbce5e9-7993-4520-9f3e-a7ece6c39d84" _VALIDATION_FILE = "file-7dbce5e9-7553-4520-9f3e-a7ece6c39d84" _FROM_CHECKPOINT = "ft-12345678-1234-1234-1234-1234567890ab" +_DUMMY_ID = "ft-12345678-1234-1234-1234-1234567890ab" _MODEL_LIMITS = FinetuneTrainingLimits( max_num_epochs=20, max_learning_rate=1.0, @@ -55,19 +56,21 @@ def mock_request(options: TogetherRequest, *args, **kwargs): return ( TogetherResponse( data={ - "id": "ft-12345678-1234-1234-1234-1234567890ab", + "id": _DUMMY_ID, }, headers={}, ), None, None, ) - else: + elif options.url == "fine-tunes/models/limits": return ( TogetherResponse(data=_MODEL_LIMITS.model_dump(), headers={}), None, None, ) + else: + raise ValueError(f"Unknown URL: {options.url}") def test_simple_request(): @@ -376,8 +379,13 @@ def test_train_on_inputs_not_supported_for_dpo(): ) -@patch("together.abstract.api_requestor.APIRequestor.request") def test_price_estimation_request(mocker): + mock_requestor = Mock() + mock_requestor.request = MagicMock() + mock_requestor.request.side_effect = mock_request + mocker.patch( + "together.abstract.api_requestor.APIRequestor", return_value=mock_requestor + ) test_data = [ { "training_type": "lora", @@ -392,20 +400,6 @@ def test_price_estimation_request(mocker): "training_method": "sft", }, ] - mocker.return_value = ( - TogetherResponse( - data={ - "estimated_total_price": 100, - "allowed_to_proceed": True, - "estimated_train_token_count": 1000, - "estimated_eval_token_count": 100, - "user_limit": 1000, - }, - headers={}, - ), - None, - None, - ) client = Together(api_key="fake_api_key") for test_case in test_data: response = client.fine_tuning.estimate_price( @@ -443,7 +437,7 @@ def test_create_ft_job(mocker): ) assert mock_requestor.request.call_count == 3 - assert response.id == "ft-12345678-1234-1234-1234-1234567890ab" + assert response.id == _DUMMY_ID response = client.fine_tuning.create( training_file=_TRAINING_FILE, @@ -457,4 +451,4 @@ def test_create_ft_job(mocker): ) assert mock_requestor.request.call_count == 5 - assert response.id == "ft-12345678-1234-1234-1234-1234567890ab" + assert response.id == _DUMMY_ID From aa0d8e783c94b1ce833cfe5da7871f0fd0bf7eea Mon Sep 17 00:00:00 2001 From: Ruslan Khaidurov Date: Wed, 3 Dec 2025 16:05:58 -0800 Subject: [PATCH 11/11] Apply suggestions from code review Co-authored-by: Max Ryabinin --- src/together/cli/api/finetune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/together/cli/api/finetune.py b/src/together/cli/api/finetune.py index f84af6f..dbd2eed 100644 --- a/src/together/cli/api/finetune.py +++ b/src/together/cli/api/finetune.py @@ -34,7 +34,7 @@ "The estimated price of this job is {price}. " "The actual cost of your job will be determined by the model size, the number of tokens " "in the training file, the number of tokens in the validation file, the number of epochs, and " - "the number of evaluations. Visit https://www.together.ai/pricing to learn more about pricing.\n" + "the number of evaluations. Visit https://www.together.ai/pricing to learn more about fine-tuning pricing.\n" "{warning}" "You can pass `-y` or `--confirm` to your command to skip this message.\n\n" "Do you want to proceed?"