config.h

#pragma once
 
#include "fwd.h"
#include "common.h"
#include "patchable_field.h"
 
#include <library/cpp/yt/misc/enum.h>
 
#include <library/cpp/yt/yson_string/public.h>
 
#include <library/cpp/yson/node/node.h>
 
#include <util/generic/maybe.h>
#include <util/generic/string.h>
#include <util/generic/hash.h>
#include <util/generic/hash_set.h>
 
#include <util/datetime/base.h>
 
namespace NYT {

 
namespace NYson {

 
struct IYsonConsumer;
 
enum class EYsonFormat : int;

 
} // namespace NYson

 
namespace NLogLevel {
    inline constexpr std::string_view Fatal = "fatal";
    inline constexpr std::string_view Error = "error";
    inline constexpr std::string_view Info = "info";
    inline constexpr std::string_view Debug = "debug";
} // namespace NLogLevel

 
extern const TString DefaultHosts;
extern const TString DefaultRemoteTempTablesDirectory;
extern const TString DefaultRemoteTempFilesDirectory;

 
enum EEncoding : int
{
    E_IDENTITY  /* "identity" */,
    E_GZIP      /* "gzip" */,
    E_BROTLI    /* "br" */,
    E_Z_LZ4     /* "z-lz4" */,
};
 
enum class ENodeReaderFormat : int
{
    Yson,  // Always use YSON format,
    Skiff, // Always use Skiff format, throw exception if it's not possible (non-strict schema, dynamic table etc.)
    Auto,  // Use Skiff format if it's possible, YSON otherwise
};
 
enum class ETraceHttpRequestsMode
{
    // Never dump http requests.
    Never /* "never" */,
    // Dump failed http requests.
    Error /* "error" */,
    // Dump all http requests.
    Always /* "always" */,
};
 
DEFINE_ENUM(EUploadDeduplicationMode,
    // For each file only one process' thread from all possible hosts can upload it to the file cache at the same time.
    // The others will wait for the uploading to finish and use already cached file.
    ((Global)   (0))
 
    // For each file and each particular host only one process' thread can upload it to the file cache at the same time.
    // The others will wait for the uploading to finish and use already cached file.
    ((Host)     (1))
 
    // All processes' threads will upload a file to the cache concurrently.
    ((Disabled) (2))
);


enum class ETableWriterVersion
{
    Auto,

    V1,

    V2,
};

 
struct TConfig
    : public TThrRefBase
{
    TString Hosts;
    TString Pool;
    TString Token;
    TString Prefix;
    TString ApiVersion;
    TString LogLevel;
    TString LogPath;
    THashSet<TString> LogExcludeCategories = {"Bus", "Net", "Dns", "Concurrency"};

    TString StructuredLog;

    TString HttpProxyRole;

    TString RpcProxyRole;

    THashMap<TString, TString> ProxyUrlAliasingRules;

    bool LogUseCore = true;
 
    // Compression for data that is sent to YT cluster.
    EEncoding ContentEncoding;
 
    // Compression for data that is read from YT cluster.
    EEncoding AcceptEncoding;
 
    TString GlobalTxId;
 
    bool ForceIpV4;
    bool ForceIpV6;
    bool UseHosts;

    bool PreferHttps;
 
    bool UseTLS;
 
    TDuration HostListUpdateInterval;
 
    TNode Spec;
    TNode TableWriter;
 
    TDuration ConnectTimeout;
    TDuration SocketTimeout;
    TDuration AddressCacheExpirationTimeout;
    TDuration TxTimeout;
    TDuration PingTimeout;
    TDuration PingInterval;
 
    int AsyncHttpClientThreads;
    int AsyncTxPingerPoolThreads;
 
    // How often should we poll for lock state
    TDuration WaitLockPollInterval;
 
    TDuration RetryInterval;
    TDuration ChunkErrorsRetryInterval;
 
    TDuration RateLimitExceededRetryInterval;
    TDuration StartOperationRetryInterval;
 
    int RetryCount;
    int ReadRetryCount;
    int StartOperationRetryCount;
 
    // The CheckClusterLiveness operation should be retried, but fewer times than other operations.
    int CheckLivenessRetryCount;

    TDuration OperationTrackerPollPeriod = TDuration::Seconds(5);
 
    TString RemoteTempFilesDirectory;
    TString RemoteTempTablesDirectory;
    // @brief Keep temp tables produced by TTempTable (despite their name). Should not be used in user programs,
    // but may be useful for setting via environment variable for debugging purposes.
    bool KeepTempTables = false;
 
    //
    // Infer schemas for nonexstent tables from typed rows (e.g. protobuf)
    // when writing from operation or client writer.
    // This options can be overridden in TOperationOptions and TTableWriterOptions.
    bool InferTableSchema;
 
    bool UseClientProtobuf;
    ENodeReaderFormat NodeReaderFormat;
    bool ProtobufFormatWithDescriptors;
 
    int ConnectionPoolSize;

    TPatchableField<i64> FileCacheReplicationFactor = TPatchableField<i64>("file_cache_replication_factor", 10);

    TDuration CacheLockTimeoutPerGb;

    EUploadDeduplicationMode CacheUploadDeduplicationMode;
 
    // @brief Minimum byte size for files to undergo deduplication at upload
    i64 CacheUploadDeduplicationThreshold;

    bool LockFileStorage = false;
 
    bool MountSandboxInTmpfs;

    TRichYPath ApiFilePathOptions;
 
    // Testing options, should never be used in user programs.
    bool UseAbortableResponse = false;
    bool EnableDebugMetrics = false;

    bool UseHaltingResponse = false;

    i64 HaltingResponseBytesLimit = 64 * 1024;
 
    //
    // There is optimization used with local YT that enables to skip binary upload and use real binary path.
    // When EnableLocalModeOptimization is set to false this optimization is completely disabled.
    bool EnableLocalModeOptimization = true;
 
    //
    // If you want see stderr even if you jobs not failed set this true.
    bool WriteStderrSuccessfulJobs = false;
 
    //
    // This configuration is useful for debug.
    // If set to ETraceHttpRequestsMode::Error library will dump all http error requests.
    // If set to ETraceHttpRequestsMode::All library will dump all http requests.
    // All tracing occurres as DEBUG level logging.
    ETraceHttpRequestsMode TraceHttpRequestsMode = ETraceHttpRequestsMode::Never;
 
    TString SkynetApiHost;
 
    // Sets SO_PRIORITY option on the socket
    TMaybe<int> SocketPriority;
 
    // Framing settings
    // (cf. https://ytsaurus.tech/docs/en/user-guide/proxy/http-reference#framing).
    THashSet<TString> CommandsWithFraming;

    ETableWriterVersion TableWriterVersion = ETableWriterVersion::Auto;

    bool RedirectStdoutToStderr = false;

    bool EnableDebugCommandLineArguments = true;

    TString ConfigRemotePatchPath = "//sys/client_config";

    TPatchableField<TString> OperationLinkPattern = TPatchableField<TString>("operation_link_pattern", "https://yt.yandex-team.ru/{cluster_ui_host}/operations/{operation_id}");

    bool EnableClientTracing = true;

    //  and the native client sends lightweight control requests via a separate multiplexing band.
    bool EnableControlMultiplexingBand = false;
 
    static bool GetBool(const char* var, bool defaultValue = false);
    static int GetInt(const char* var, int defaultValue);
    static TDuration GetDuration(const char* var, TDuration defaultValue);
    static EEncoding GetEncoding(const char* var);
    static EUploadDeduplicationMode GetUploadingDeduplicationMode(
        const char* var,
        EUploadDeduplicationMode defaultValue);
 
    static void ValidateToken(const TString& token);
    static TString LoadTokenFromFile(const TString& tokenPath);
 
    static TNode LoadJsonSpec(const TString& strSpec);
 
    static TRichYPath LoadApiFilePathOptions(const TString& ysonMap);
 
    void LoadToken();
    void LoadSpec();
    void LoadTimings();
    void LoadProxyUrlAliasingRules();
 
    void Reset();
 
    TConfig();
 
    static TConfigPtr Get();
};

 
void Serialize(const TConfig& config, NYson::IYsonConsumer* consumer);
 
void Deserialize(TConfig& config, const TNode& node);

 
TString ConfigToYsonString(const TConfig& config, NYson::EYsonFormat format = NYson::EYsonFormat::Pretty);
 
TConfig ConfigFromYsonString(TString serializedConfig);

 
struct TProcessState
{
    TString FqdnHostName;
    TString UserName;
 
    int Pid;
    TString ClientVersion;
    TString BinaryPath;
    TString BinaryName;
 
    TProcessState();
 
    static TProcessState* Get();
};

 
} // namespace NYT