warn users when they use unsupported options (#386)

* warn users about unsupported options

* do the rest

* don't report that Warn() isn't tested
This commit is contained in:
Boris Verkhovskiy
2022-04-16 21:56:33 -07:00
committed by GitHub
parent d83919e85c
commit 467f093a72
23 changed files with 1086 additions and 160 deletions

View File

@@ -11,23 +11,32 @@ import {
} from "./util.js";
import type { LongOpts, ShortOpts, Request } from "./util.js";
import { _toAnsible } from "./generators/ansible.js";
import { _toDart } from "./generators/dart.js";
import { _toCFML } from "./generators/cfml.js";
import { _toElixir } from "./generators/elixir.js";
import { _toGo } from "./generators/go.js";
import { _toJava } from "./generators/java.js";
import { _toJavaScript } from "./generators/javascript/javascript.js";
import { _toJsonString } from "./generators/json.js";
import { _toMATLAB } from "./generators/matlab/matlab.js";
import { _toNode } from "./generators/javascript/node-fetch.js";
import { _toNodeRequest } from "./generators/javascript/node-request.js";
import { _toPhp } from "./generators/php/php.js";
import { _toPhpRequests } from "./generators/php/php-requests.js";
import { _toPython } from "./generators/python.js";
import { _toR } from "./generators/r.js";
import { _toRust } from "./generators/rust.js";
import { _toStrest } from "./generators/strest.js";
import { _toAnsible, toAnsibleWarn } from "./generators/ansible.js";
import { _toDart, toDartWarn } from "./generators/dart.js";
import { _toCFML, toCFMLWarn } from "./generators/cfml.js";
import { _toElixir, toElixirWarn } from "./generators/elixir.js";
import { _toGo, toGoWarn } from "./generators/go.js";
import { _toJava, toJavaWarn } from "./generators/java.js";
import {
_toJavaScript,
toJavaScriptWarn,
} from "./generators/javascript/javascript.js";
import { _toJsonString, toJsonStringWarn } from "./generators/json.js";
import { _toMATLAB, toMATLABWarn } from "./generators/matlab/matlab.js";
import { _toNode, toNodeWarn } from "./generators/javascript/javascript.js";
import {
_toNodeRequest,
toNodeRequestWarn,
} from "./generators/javascript/node-request.js";
import { _toPhp, toPhpWarn } from "./generators/php/php.js";
import {
_toPhpRequests,
toPhpRequestsWarn,
} from "./generators/php/php-requests.js";
import { _toPython, toPythonWarn } from "./generators/python.js";
import { _toR, toRWarn } from "./generators/r.js";
import { _toRust, toRustWarn } from "./generators/rust.js";
import { _toStrest, toStrestWarn } from "./generators/strest.js";
import fs from "fs";
@@ -39,25 +48,30 @@ const defaultLanguage = "python";
// Maps options for --language to functions
// NOTE: make sure to update this when adding language support
const translate: { [key: string]: (request: Request) => string } = {
ansible: _toAnsible,
cfml: _toCFML,
browser: _toJavaScript, // for backwards compatibility, undocumented
dart: _toDart,
elixir: _toElixir,
go: _toGo,
java: _toJava,
javascript: _toJavaScript,
json: _toJsonString,
matlab: _toMATLAB,
node: _toNode,
"node-request": _toNodeRequest,
php: _toPhp,
"php-requests": _toPhpRequests,
python: _toPython,
r: _toR,
rust: _toRust,
strest: _toStrest,
const translate: {
[key: string]: [
(request: Request) => string,
(curlCommand: string | string[]) => [string, [string, string][]]
];
} = {
ansible: [_toAnsible, toAnsibleWarn],
cfml: [_toCFML, toCFMLWarn],
browser: [_toJavaScript, toJavaScriptWarn], // for backwards compatibility, undocumented
dart: [_toDart, toDartWarn],
elixir: [_toElixir, toElixirWarn],
go: [_toGo, toGoWarn],
java: [_toJava, toJavaWarn],
javascript: [_toJavaScript, toJavaScriptWarn],
json: [_toJsonString, toJsonStringWarn],
matlab: [_toMATLAB, toMATLABWarn],
node: [_toNode, toNodeWarn],
"node-request": [_toNodeRequest, toNodeRequestWarn],
php: [_toPhp, toPhpWarn],
"php-requests": [_toPhpRequests, toPhpRequestsWarn],
python: [_toPython, toPythonWarn],
r: [_toR, toRWarn],
rust: [_toRust, toRustWarn],
strest: [_toStrest, toStrestWarn],
};
const USAGE = `Usage: curlconverter [--language <language>] [-] [curl_options...]
@@ -94,10 +108,8 @@ const curlConverterShortOpts: ShortOpts = {
// a single - (dash) tells curlconverter to read input from stdin
"": "stdin",
};
const opts: [LongOpts, ShortOpts] = [
{ ...curlLongOpts, ...curlConverterLongOpts },
{ ...curlShortOpts, ...curlConverterShortOpts },
];
const longOpts: LongOpts = { ...curlLongOpts, ...curlConverterLongOpts };
const shortOpts: ShortOpts = { ...curlShortOpts, ...curlConverterShortOpts };
function exitWithError(error: unknown, verbose = false): never {
let errMsg: Error | string | unknown = error;
@@ -118,9 +130,9 @@ function exitWithError(error: unknown, verbose = false): never {
}
const argv = process.argv.slice(2);
let parsedArguments;
let parsedArguments, warnings;
try {
parsedArguments = parseArgs(argv, opts);
[parsedArguments, warnings] = parseArgs(argv, longOpts, shortOpts);
} catch (e) {
exitWithError(e);
}
@@ -152,12 +164,14 @@ for (const opt of Object.keys(curlConverterLongOpts)) {
delete parsedArguments[opt];
}
let request;
const [generator, warnGenerator] = translate[language];
let code;
if (argc === 0) {
console.log(USAGE.trim());
process.exit(2);
} else if (stdin) {
// This lets you do something like
}
if (stdin) {
// This lets you do
// echo curl example.com | curlconverter --verbose
const extraArgs = Object.keys(parsedArguments).filter((a) => a !== "verbose");
if (extraArgs.length > 0) {
@@ -174,34 +188,39 @@ if (argc === 0) {
}
const input = fs.readFileSync(0, "utf8");
try {
request = parseCurlCommand(input);
[code, warnings] = warnGenerator(input);
} catch (e) {
exitWithError(e, parsedArguments.verbose);
}
} else {
let request;
try {
request = buildRequest(parsedArguments);
} catch (e) {
exitWithError(e, parsedArguments.verbose);
}
// Warning for users using the pre-4.0 CLI
if (request.url?.startsWith("curl ")) {
console.error(
"warning: Passing a whole curl command as a single argument?"
);
console.error(
"warning: Pass options to curlconverter as if it was curl instead:"
);
console.error(
"warning: curlconverter 'curl example.com' -> curlconverter example.com"
);
}
try {
code = generator(request);
} catch (e) {
exitWithError(e, parsedArguments.verbose);
}
}
// Warning for users using the pre-4.0 CLI
if (request.url?.startsWith("curl ")) {
console.error("warning: Passing a whole curl command as a single argument?");
console.error(
"warning: Pass options to curlconverter as if it was curl instead:"
);
console.error(
"warning: curlconverter 'curl example.com' -> curlconverter example.com"
);
}
const generator = translate[language];
let code;
try {
code = generator(request);
} catch (e) {
exitWithError(e, parsedArguments.verbose);
if (warnings && parsedArguments.verbose) {
for (const w of warnings) {
console.error("warning: " + w[1]);
}
}
process.stdout.write(code);

View File

@@ -1,10 +1,33 @@
import * as util from "../util.js";
import type { Request } from "../util.js";
import type { Request, Warnings } from "../util.js";
import { ansibleTemplate } from "../templates/ansible.js";
import nunjucks from "nunjucks";
import querystring from "query-string";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
// "form",
// "form-string",
"get",
"header",
"head",
"no-head",
"insecure",
"no-insecure",
"user",
]);
function getDataString(request: Request): string | object {
if (!request.data) {
return "";
@@ -34,7 +57,13 @@ export const _toAnsible = (request: Request): string => {
});
return result;
};
export const toAnsibleWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toAnsible(request), warnings];
};
export const toAnsible = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toAnsible(request);
};

View File

@@ -1,8 +1,32 @@
import * as util from "../util.js";
import type { Request } from "../util.js";
import type { Request, Warnings } from "../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
"form",
"form-string",
"get",
"header",
"head",
"no-head",
"user",
"proxy-user",
"proxy",
"max-time",
]);
const quote = (str: string): string => {
return jsesc(str, { quotes: "single" }).replace(/"/g, '""');
};
@@ -112,7 +136,14 @@ export const _toCFML = (request: Request): string => {
return cfmlCode;
};
export const toCFMLWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toCFML(request), warnings];
};
export const toCFML = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toCFML(request);
};

View File

@@ -1,8 +1,30 @@
import * as util from "../util.js";
import type { Request } from "../util.js";
import type { Request, Warnings } from "../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"compressed",
"no-compressed",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
// "form",
// "form-string",
"get",
"header",
"head",
"no-head",
"user",
]);
function repr(value: string): string {
// In context of url parameters, don't accept nulls and such.
if (!value) {
@@ -120,7 +142,13 @@ export const _toDart = (r: Request): string => {
return s + "\n";
};
export const toDart = (curlCommand: string | string[]): string => {
const r = util.parseCurlCommand(curlCommand);
return _toDart(r);
export const toDartWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toDart(request), warnings];
};
export const toDart = (curlCommand: string | string[]): string => {
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toDart(request);
};

View File

@@ -1,9 +1,32 @@
import * as util from "../util.js";
import type { Request } from "../util.js";
import type { Request, Warnings } from "../util.js";
import jsesc from "jsesc";
import querystring from "query-string";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
"form",
"form-string",
"get",
"header",
"head",
"no-head",
"insecure",
"no-insecure",
"user",
]);
// TODO: I bet elixir's array syntax is different and if the query string
// values are arrays that actually generates broken code.
function repr(value: string | null | (string | null)[]): string {
@@ -230,7 +253,14 @@ response = HTTPoison.request(request)
return template;
};
export const toElixirWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toElixir(request), warnings];
};
export const toElixir = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toElixir(request);
};

View File

@@ -1,8 +1,30 @@
import * as util from "../util.js";
import type { Request } from "../util.js";
import type { Request, Warnings } from "../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
// TODO
// "form",
// "form-string",
"get",
"header",
"head",
"no-head",
"user",
]);
const reprMaybeBacktick = (s: string): string => {
return s.includes('"') && !s.includes("`") ? reprBacktick(s) : repr(s);
};
@@ -70,7 +92,13 @@ export const _toGo = (request: Request): string => {
return goCode + "\n";
};
export const toGoWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toGo(request), warnings];
};
export const toGo = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toGo(request);
};

View File

@@ -1,8 +1,30 @@
import * as util from "../util.js";
import type { Request } from "../util.js";
import type { Request, Warnings } from "../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
// TODO
// "form",
// "form-string",
"get",
"header",
"head",
"no-head",
"user",
]);
const doubleQuotes = (str: string): string => jsesc(str, { quotes: "double" });
export const _toJava = (request: Request): string => {
@@ -92,7 +114,14 @@ export const _toJava = (request: Request): string => {
return javaCode + "\n";
};
export const toJavaWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toJava(request), warnings];
};
export const toJava = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toJava(request);
};

View File

@@ -1,8 +1,30 @@
import * as util from "../../util.js";
import type { Warnings } from "../../util.js";
import type { Request } from "../../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
// "form",
// "form-string",
"get",
"header",
"head",
"no-head",
"user",
]);
export const _toJavaScript = (request: Request): string => {
let jsFetchCode = "";
@@ -88,7 +110,31 @@ export const _toJavaScript = (request: Request): string => {
return jsFetchCode + "\n";
};
export const toJavaScriptWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toJavaScript(request), warnings];
};
export const toJavaScript = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return _toJavaScript(request);
};
const importStatement = "var fetch = require('node-fetch');\n\n";
export const _toNode = (request: Request): string => {
return importStatement + _toJavaScript(request);
};
export const toNodeWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toNode(request), warnings];
};
export const toNode = (curlCommand: string | string[]): string => {
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toNode(request);
};

View File

@@ -1,13 +0,0 @@
import * as util from "../../util.js";
import type { Request } from "../../util.js";
import { _toJavaScript } from "./javascript.js";
const importStatement = "var fetch = require('node-fetch');\n\n";
export const _toNode = (request: Request): string => {
return importStatement + _toJavaScript(request);
};
export const toNode = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
return _toNode(request);
};

View File

@@ -1,8 +1,29 @@
import * as util from "../../util.js";
import type { Request } from "../../util.js";
import type { Request, Warnings } from "../../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
// "form",
// "form-string",
"get",
"header",
"head",
"no-head",
"user",
]);
export const _toNodeRequest = (request: Request): string => {
let nodeRequestCode = "var request = require('request');\n\n";
if (request.headers) {
@@ -64,7 +85,13 @@ export const _toNodeRequest = (request: Request): string => {
return nodeRequestCode + "\n";
};
export const toNodeRequestWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toNodeRequest(request), warnings];
};
export const toNodeRequest = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toNodeRequest(request);
};

View File

@@ -1,10 +1,33 @@
// Author: ssi-anik (sirajul.islam.anik@gmail.com)
import * as util from "../util.js";
import type { Request, QueryDict } from "../util.js";
import type { Request, QueryDict, Warnings } from "../util.js";
import querystring from "query-string";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
"form",
"form-string",
"get",
"header",
"head",
"no-head",
"insecure",
"no-insecure",
"user",
]);
type JSONOutput = {
url: string;
raw_url: string;
@@ -165,7 +188,13 @@ export const _toJsonString = (request: Request) => {
) + "\n"
);
};
export const toJsonString = (curlCommand: string | string[]) => {
const request = util.parseCurlCommand(curlCommand);
export const toJsonStringWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toJsonString(request), warnings];
};
export const toJsonString = (curlCommand: string | string[]): string => {
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toJsonString(request);
};

View File

@@ -1,9 +1,50 @@
import * as util from "../../util.js";
import type { Request } from "../../util.js";
import type { Request, Warnings } from "../../util.js";
import { toWebServices } from "./webservices.js";
import { toHTTPInterface } from "./httpinterface.js";
const supportedArgs = new Set([
"url",
"request",
"compressed",
"no-compressed",
"digest",
"no-digest",
"http1.0",
"http1.1",
"http2",
"http2-prior-knowledge",
"http3",
"http0.9",
"no-http0.9",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
"cert",
"cacert",
"key",
"capath",
"form",
"form-string",
"get",
"header",
"head",
"no-head",
"insecure",
"no-insecure",
"output",
"user",
"proxy-user",
"proxy",
]);
export const _toMATLAB = (request: Request): string => {
const lines = toWebServices(request).concat("", toHTTPInterface(request));
return lines
@@ -11,7 +52,13 @@ export const _toMATLAB = (request: Request): string => {
.filter((line) => line !== null)
.join("\n");
};
export const toMATLABWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toMATLAB(request), warnings];
};
export const toMATLAB = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toMATLAB(request);
};

View File

@@ -1,9 +1,30 @@
import * as util from "../../util.js";
import type { Request } from "../../util.js";
import type { Request, Warnings } from "../../util.js";
import querystring from "query-string";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
// "form",
// "form-string",
"get",
"header",
"head",
"no-head",
"user",
]);
// TODO: only string
const quote = (str: string | null | (string | null)[]): string =>
jsesc(str, { quotes: "single" });
@@ -90,7 +111,13 @@ export const _toPhpRequests = (request: Request): string => {
return phpCode + "\n";
};
export const toPhpRequestsWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toPhpRequests(request), warnings];
};
export const toPhpRequests = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toPhpRequests(request);
};

View File

@@ -1,8 +1,39 @@
import * as util from "../../util.js";
import type { Request } from "../../util.js";
import type { Request, Warnings } from "../../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"compressed",
"no-compressed",
"digest",
"no-digest",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
"form",
"form-string",
"get",
"header",
"head",
"no-head",
"insecure",
"no-insecure",
"user",
"proxy-user",
"proxy",
"max-time",
"location",
]);
const quote = (str: string): string => jsesc(str, { quotes: "single" });
export const _toPhp = (request: Request): string => {
@@ -129,7 +160,13 @@ export const _toPhp = (request: Request): string => {
return phpCode;
};
export const toPhpWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toPhp(request), warnings];
};
export const toPhp = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toPhp(request);
};

View File

@@ -1,8 +1,358 @@
import * as util from "../util.js";
import type { Warnings } from "../util.js";
import jsesc from "jsesc";
import type { Request, Query, QueryDict } from "../util.js";
// TODO: partiallySupportedArgs
const supportedArgs = new Set([
"url",
"request",
"compressed",
"no-compressed",
"digest",
"no-digest",
"http1.0",
"http1.1",
"http2",
"http2-prior-knowledge",
"http3",
"http0.9",
"no-http0.9",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
"cert",
"cacert",
"key",
"capath",
"form",
"form-string",
"get",
"header",
"head",
"no-head",
"insecure",
"no-insecure",
"output",
"user",
"upload-file",
"proxy-user",
"proxy",
]);
// supported by other generators
const supportedByOthers = ["max-time", "location"];
const unsupportedArgs = [
"dns-ipv4-addr",
"dns-ipv6-addr",
"random-file",
"egd-file",
"oauth2-bearer",
"connect-timeout",
"doh-url",
"ciphers",
"dns-interface",
"disable-epsv",
"no-disable-epsv",
"disallow-username-in-url",
"no-disallow-username-in-url",
"epsv",
"no-epsv",
"dns-servers",
"trace",
"npn",
"no-npn",
"trace-ascii",
"alpn",
"no-alpn",
"limit-rate",
"tr-encoding",
"no-tr-encoding",
"negotiate",
"no-negotiate",
"ntlm",
"no-ntlm",
"ntlm-wb",
"no-ntlm-wb",
"basic",
"no-basic",
"anyauth",
"no-anyauth",
"wdebug",
"no-wdebug",
"ftp-create-dirs",
"no-ftp-create-dirs",
"create-dirs",
"no-create-dirs",
"create-file-mode",
"max-redirs",
"proxy-ntlm",
"no-proxy-ntlm",
"crlf",
"no-crlf",
"stderr",
"aws-sigv4",
"interface",
"krb",
"krb4",
"haproxy-protocol",
"no-haproxy-protocol",
"max-filesize",
"disable-eprt",
"no-disable-eprt",
"eprt",
"no-eprt",
"xattr",
"no-xattr",
"ftp-ssl",
"no-ftp-ssl",
"ssl",
"no-ssl",
"ftp-pasv",
"no-ftp-pasv",
"socks5",
"tcp-nodelay",
"no-tcp-nodelay",
"proxy-digest",
"no-proxy-digest",
"proxy-basic",
"no-proxy-basic",
"retry",
"retry-connrefused",
"no-retry-connrefused",
"retry-delay",
"retry-max-time",
"proxy-negotiate",
"no-proxy-negotiate",
"form-escape",
"no-form-escape",
"ftp-account",
"proxy-anyauth",
"no-proxy-anyauth",
"trace-time",
"no-trace-time",
"ignore-content-length",
"no-ignore-content-length",
"ftp-skip-pasv-ip",
"no-ftp-skip-pasv-ip",
"ftp-method",
"local-port",
"socks4",
"socks4a",
"ftp-alternative-to-user",
"ftp-ssl-reqd",
"no-ftp-ssl-reqd",
"ssl-reqd",
"no-ssl-reqd",
"sessionid",
"no-sessionid",
"ftp-ssl-control",
"no-ftp-ssl-control",
"ftp-ssl-ccc",
"no-ftp-ssl-ccc",
"ftp-ssl-ccc-mode",
"libcurl",
"raw",
"no-raw",
"post301",
"no-post301",
"keepalive",
"no-keepalive",
"socks5-hostname",
"keepalive-time",
"post302",
"no-post302",
"noproxy",
"socks5-gssapi-nec",
"no-socks5-gssapi-nec",
"proxy1.0",
"tftp-blksize",
"mail-from",
"mail-rcpt",
"ftp-pret",
"no-ftp-pret",
"proto",
"proto-redir",
"resolve",
"delegation",
"mail-auth",
"post303",
"no-post303",
"metalink",
"no-metalink",
"sasl-authzid",
"sasl-ir",
"no-sasl-ir",
"test-event",
"no-test-event",
"unix-socket",
"path-as-is",
"no-path-as-is",
"socks5-gssapi-service",
"proxy-service-name",
"service-name",
"proto-default",
"expect100-timeout",
"tftp-no-options",
"no-tftp-no-options",
"connect-to",
"abstract-unix-socket",
"tls-max",
"suppress-connect-headers",
"no-suppress-connect-headers",
"compressed-ssh",
"no-compressed-ssh",
"happy-eyeballs-timeout-ms",
"retry-all-errors",
"no-retry-all-errors",
"tlsv1",
"tlsv1.0",
"tlsv1.1",
"tlsv1.2",
"tlsv1.3",
"tls13-ciphers",
"proxy-tls13-ciphers",
"sslv2",
"sslv3",
"ipv4",
"ipv6",
"append",
"no-append",
"alt-svc",
"hsts",
"use-ascii",
"no-use-ascii",
"cookie-jar",
"continue-at",
"dump-header",
"cert-type",
"key-type",
"pass",
"engine",
"pubkey",
"hostpubmd5",
"hostpubsha256",
"crlfile",
"tlsuser",
"tlspassword",
"tlsauthtype",
"ssl-allow-beast",
"no-ssl-allow-beast",
"ssl-auto-client-cert",
"no-ssl-auto-client-cert",
"proxy-ssl-auto-client-cert",
"no-proxy-ssl-auto-client-cert",
"pinnedpubkey",
"proxy-pinnedpubkey",
"cert-status",
"no-cert-status",
"doh-cert-status",
"no-doh-cert-status",
"false-start",
"no-false-start",
"ssl-no-revoke",
"no-ssl-no-revoke",
"ssl-revoke-best-effort",
"no-ssl-revoke-best-effort",
"tcp-fastopen",
"no-tcp-fastopen",
"proxy-tlsuser",
"proxy-tlspassword",
"proxy-tlsauthtype",
"proxy-cert",
"proxy-cert-type",
"proxy-key",
"proxy-key-type",
"proxy-pass",
"proxy-ciphers",
"proxy-crlfile",
"proxy-ssl-allow-beast",
"no-proxy-ssl-allow-beast",
"login-options",
"proxy-cacert",
"proxy-capath",
"proxy-insecure",
"no-proxy-insecure",
"proxy-tlsv1",
"socks5-basic",
"no-socks5-basic",
"socks5-gssapi",
"no-socks5-gssapi",
"etag-save",
"etag-compare",
"curves",
"fail",
"no-fail",
"fail-early",
"no-fail-early",
"styled-output",
"no-styled-output",
"mail-rcpt-allowfails",
"no-mail-rcpt-allowfails",
"fail-with-body",
"no-fail-with-body",
"globoff",
"no-globoff",
"request-target",
"proxy-header",
"include",
"no-include",
"junk-session-cookies",
"no-junk-session-cookies",
"remote-header-name",
"no-remote-header-name",
"doh-insecure",
"no-doh-insecure",
"config",
"list-only",
"no-list-only",
"location",
"no-location",
"location-trusted",
"no-location-trusted",
"max-time",
"manual",
"no-manual",
"netrc",
"no-netrc",
"netrc-optional",
"no-netrc-optional",
"netrc-file",
"buffer",
"no-buffer",
"remote-name",
"remote-name-all",
"no-remote-name-all",
"output-dir",
"proxytunnel",
"no-proxytunnel",
"ftp-port",
"disable",
"no-disable",
"quote",
"range",
"remote-time",
"no-remote-time",
"telnet-option",
"write-out",
"preproxy",
"speed-limit",
"speed-time",
"time-cond",
"parallel",
"no-parallel",
"parallel-max",
"parallel-immediate",
"no-parallel-immediate",
"next", // TODO: this is a big one
];
function reprWithVariable(value: string, hasEnvironmentVariable: boolean) {
if (!value) {
return "''";
@@ -644,7 +994,15 @@ export const _toPython = (request: Request): string => {
return pythonCode + "\n";
};
export const toPythonWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toPython(request), warnings];
};
export const toPython = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toPython(request);
};

View File

@@ -1,10 +1,32 @@
// Author: Bob Rudis (bob@rud.is)
import * as util from "../util.js";
import type { Request, Cookie, QueryDict } from "../util.js";
import type { Request, Cookie, QueryDict, Warnings } from "../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
"form",
"form-string",
"get",
"header",
"head",
"no-head",
"insecure",
"no-insecure",
"user",
]);
function reprn(value: string | null): string {
// back-tick quote names
if (!value) {
@@ -207,7 +229,11 @@ export const _toR = (request: Request) => {
return rstatsCode + "\n";
};
export const toRWarn = (curlCommand: string | string[]): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toR(request), warnings];
};
export const toR = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toR(request);
};

View File

@@ -1,8 +1,29 @@
import * as util from "../util.js";
import type { Request } from "../util.js";
import type { Request, Warnings } from "../util.js";
import jsesc from "jsesc";
const supportedArgs = new Set([
"url",
"request",
"user-agent",
"cookie",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"referer",
"form",
"form-string",
"get",
"header",
"head",
"no-head",
"user",
]);
const INDENTATION = " ".repeat(4);
const indent = (line: string, level = 1): string =>
INDENTATION.repeat(level) + line;
@@ -99,7 +120,13 @@ export const _toRust = (request: Request) => {
return lines.join("\n") + "\n";
};
export const toRust = (curlCommand: string | string[]) => {
const request = util.parseCurlCommand(curlCommand);
export const toRustWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toRust(request), warnings];
};
export const toRust = (curlCommand: string | string[]): string => {
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toRust(request);
};

View File

@@ -1,10 +1,31 @@
import * as util from "../util.js";
import type { Request } from "../util.js";
import type { Request, Warnings } from "../util.js";
import yaml from "yamljs";
import jsesc from "jsesc";
import querystring from "query-string";
const supportedArgs = new Set([
"url",
"request",
"data",
"data-raw",
"data-ascii",
"data-binary",
"data-urlencode",
"json",
"user-agent",
"cookie",
"referer",
"header",
"get",
"head",
"no-head",
"user",
"insecure",
"no-insecure",
]);
function getDataString(request: Request): PostData | null {
if (!request.data) {
return null;
@@ -120,7 +141,13 @@ export const _toStrest = (request: Request): string => {
const yamlString = yaml.stringify(response, 100, 2);
return yamlString;
};
export const toStrestWarn = (
curlCommand: string | string[]
): [string, Warnings] => {
const [request, warnings] = util.parseCurlCommand(curlCommand, supportedArgs);
return [_toStrest(request), warnings];
};
export const toStrest = (curlCommand: string | string[]): string => {
const request = util.parseCurlCommand(curlCommand);
const [request, warnings] = util.parseCurlCommand(curlCommand);
return _toStrest(request);
};

View File

@@ -1,23 +1,34 @@
export { toAnsible } from "./generators/ansible.js";
export { toCFML } from "./generators/cfml.js";
export { toJavaScript } from "./generators/javascript/javascript.js";
// backwards compatibility alias
export { toAnsible, toAnsibleWarn } from "./generators/ansible.js";
export { toCFML, toCFMLWarn } from "./generators/cfml.js";
export {
toJavaScript,
toJavaScriptWarn,
} from "./generators/javascript/javascript.js";
export { toDart, toDartWarn } from "./generators/dart.js";
export { toElixir, toElixirWarn } from "./generators/elixir.js";
export { toGo, toGoWarn } from "./generators/go.js";
export { toJava, toJavaWarn } from "./generators/java.js";
export { toJsonString, toJsonStringWarn } from "./generators/json.js";
export { toMATLAB, toMATLABWarn } from "./generators/matlab/matlab.js";
export { toNode, toNodeWarn } from "./generators/javascript/javascript.js";
export {
toNodeRequest,
toNodeRequestWarn,
} from "./generators/javascript/node-request.js";
export { toPhp, toPhpWarn } from "./generators/php/php.js";
export {
toPhpRequests,
toPhpRequestsWarn,
} from "./generators/php/php-requests.js";
export { toPython, toPythonWarn } from "./generators/python.js";
export { toR, toRWarn } from "./generators/r.js";
export { toRust, toRustWarn } from "./generators/rust.js";
export { toStrest, toStrestWarn } from "./generators/strest.js";
// backwards compatibility aliases
export { toJavaScript as toBrowser } from "./generators/javascript/javascript.js";
export { toDart } from "./generators/dart.js";
export { toElixir } from "./generators/elixir.js";
export { toGo } from "./generators/go.js";
export { toJava } from "./generators/java.js";
export { toJsonString } from "./generators/json.js";
export { toMATLAB } from "./generators/matlab/matlab.js";
export { toNode } from "./generators/javascript/node-fetch.js";
// backwards compatibility alias
export { toNode as toNodeFetch } from "./generators/javascript/node-fetch.js";
export { toNodeRequest } from "./generators/javascript/node-request.js";
export { toPhp } from "./generators/php/php.js";
export { toPhpRequests } from "./generators/php/php-requests.js";
export { toPython } from "./generators/python.js";
export { toR } from "./generators/r.js";
export { toRust } from "./generators/rust.js";
export { toStrest } from "./generators/strest.js";
export { toNode as toNodeFetch } from "./generators/javascript/javascript.js";
export { toJavaScriptWarn as toBrowserWarn } from "./generators/javascript/javascript.js";
export { toNodeWarn as toNodeFetchWarn } from "./generators/javascript/javascript.js";
export { CCError } from "./util.js";

View File

@@ -63,6 +63,12 @@ interface ShortOpts {
[key: string]: string;
}
interface PossibleArgs {
supported: string[];
unsupported: string[];
ignored: string[];
}
type Query = Array<[string, string | null]>;
interface QueryDict {
[key: string]: string | null | Array<string | null>;
@@ -87,6 +93,8 @@ interface ParsedArguments {
[key: string]: any;
}
type Warnings = [string, string][];
function pushArgValue(obj: ParsedArguments, argName: string, value: string) {
if (argName === "form-string") {
return pushProp(obj, "form", { value, type: "string" });
@@ -730,6 +738,24 @@ const changedShortOpts: { [key: string]: string } = {
"~": "used to be short for --xattr until curl 7.49.0",
};
// These are args that users wouldn't expect to be warned about
const ignoredArgs = new Set([
"help",
"no-help",
"silent",
"no-silent",
"verbose",
"no-verbose",
"version",
"no-version",
"progress-bar",
"no-progress-bar",
"progress-meter",
"no-progress-meter",
"show-error",
"no-show-error",
]);
// These options can be specified more than once, they
// are always returned as a list.
// Normally, if you specify some option more than once,
@@ -1096,9 +1122,37 @@ const tokenize = (curlCommand: string): TokenizeResult => {
};
};
const parseArgs = (args: string[], opts?: [LongOpts, ShortOpts]) => {
const [longOpts, shortOpts] = opts || [curlLongOpts, curlShortOpts];
const checkLongOpt = (
lookup: string,
longArgName: string,
supportedOpts: Set<string>,
warnings: Warnings
) => {
if (!supportedOpts.has(longArgName) && !ignoredArgs.has(longArgName)) {
// TODO: better message. include generator name?
warnings.push([longArgName, "--" + lookup + " is not a supported option"]);
}
};
const checkShortOpt = (
lookup: string,
longArgName: string,
supportedOpts: Set<string>,
warnings: Warnings
) => {
if (!supportedOpts.has(longArgName) && !ignoredArgs.has(longArgName)) {
// TODO: better message. include generator name?
warnings.push([longArgName, "-" + lookup + " is not a supported option"]);
}
};
const parseArgs = (
args: string[],
longOpts: LongOpts,
shortOpts: ShortOpts,
supportedOpts?: Set<string>
): [ParsedArguments, Warnings] => {
const warnings: Warnings = [];
const parsedArguments: ParsedArguments = {};
for (let i = 0, stillflags = true; i < args.length; i++) {
let arg: string | string[] = args[i];
@@ -1109,7 +1163,8 @@ const parseArgs = (args: string[], opts?: [LongOpts, ShortOpts]) => {
following (URL) argument to start with -. */
stillflags = false;
} else if (arg.startsWith("--")) {
const longArg = longOpts[arg.slice(2)];
const lookup = arg.slice(2);
const longArg = longOpts[lookup];
if (longArg === null) {
throw new CCError("option " + arg + ": is ambiguous");
}
@@ -1128,6 +1183,9 @@ const parseArgs = (args: string[], opts?: [LongOpts, ShortOpts]) => {
} else {
parsedArguments[longArg.name] = toBoolean(arg.slice(2)); // TODO: all shortened args work correctly?
}
if (supportedOpts) {
checkLongOpt(lookup, longArg.name, supportedOpts, warnings);
}
} else {
// Short option. These can look like
// -X POST -> {request: 'POST'}
@@ -1140,7 +1198,7 @@ const parseArgs = (args: string[], opts?: [LongOpts, ShortOpts]) => {
// -ABCXPOST
// -> {A: true, B: true, C: true, request: 'POST'}
// "-" passed to curl on its own raises an error,
// "-" passed to curl as an argument raises an error,
// curlconverter's command line uses it to read from stdin
if (arg.length === 1) {
if (has(shortOpts, "")) {
@@ -1160,7 +1218,8 @@ const parseArgs = (args: string[], opts?: [LongOpts, ShortOpts]) => {
// TODO: there are a few deleted short options we could report
throw new CCError("option " + argRepr + ": is unknown");
}
const shortFor = shortOpts[arg[j]];
const lookup = arg[j];
const shortFor = shortOpts[lookup];
const longArg = longOpts[shortFor];
if (longArg === null) {
// This could happen if curlShortOpts points to a renamed option or has a typo
@@ -1184,6 +1243,9 @@ const parseArgs = (args: string[], opts?: [LongOpts, ShortOpts]) => {
// and we want to end up with {buffer: false}
parsedArguments[longArg.name] = toBoolean(shortFor);
}
if (supportedOpts && lookup) {
checkShortOpt(lookup, longArg.name, supportedOpts, warnings);
}
}
}
} else {
@@ -1196,7 +1258,7 @@ const parseArgs = (args: string[], opts?: [LongOpts, ShortOpts]) => {
parsedArguments[arg] = values[values.length - 1];
}
}
return parsedArguments;
return [parsedArguments, warnings];
};
export function parseQueryString(
@@ -1591,7 +1653,10 @@ function buildRequest(parsedArguments: ParsedArguments): Request {
return request;
}
const parseCurlCommand = (curlCommand: string | string[]): Request => {
function parseCurlCommand(
curlCommand: string | string[],
supportedArgs?: Set<string>
): [Request, Warnings] {
let cmdName: string,
args: string[],
stdin: undefined | string,
@@ -1623,7 +1688,12 @@ const parseCurlCommand = (curlCommand: string | string[]): Request => {
}
}
const parsedArguments = parseArgs(args);
const [parsedArguments, warnings] = parseArgs(
args,
curlLongOpts,
curlShortOpts,
supportedArgs
);
const request = buildRequest(parsedArguments);
if (stdin) {
request.stdin = stdin;
@@ -1631,8 +1701,8 @@ const parseCurlCommand = (curlCommand: string | string[]): Request => {
if (input) {
request.input = input;
}
return request;
};
return [request, warnings];
}
// Gets the first header, matching case-insensitively
const getHeader = (
@@ -1759,4 +1829,13 @@ export {
has,
};
export type { LongOpts, ShortOpts, Request, Cookie, Cookies, Query, QueryDict };
export type {
LongOpts,
ShortOpts,
Request,
Cookie,
Cookies,
Query,
QueryDict,
Warnings,
};

View File

@@ -10,7 +10,7 @@ export const fixturesDir = path.resolve(__dirname, "../../test/fixtures");
// Special case that returns the parsed argument object
const toParser = (curl: string | string[]): string => {
const parserOutput = utils.parseCurlCommand(curl);
const [parserOutput, warnings] = utils.parseCurlCommand(curl);
const code = JSON.stringify(parserOutput, null, 2);
return code + "\n";
};
@@ -132,7 +132,7 @@ const availableConverters = Object.entries(curlconverter)
.map((c) => c[1].name)
.filter((n) => n !== "CCError");
const missing = availableConverters.filter(
(c) => !testedConverters.includes(c)
(c) => !testedConverters.includes(c) && !c.endsWith("Warn")
);
const extra = testedConverters.filter(
(c) => !availableConverters.includes(c) && c !== "toParser"

View File

@@ -129,7 +129,7 @@ const testFile = async (testFilename: string): Promise<void> => {
throw new Error("input file doesn't exist: " + inputFile);
}
const curlCommand = fs.readFileSync(inputFile, "utf8");
const requestedUrl = utils.parseCurlCommand(curlCommand).url;
const requestedUrl = utils.parseCurlCommand(curlCommand)[0].url;
if (!requestedUrl.replace("http://", "").startsWith(EXPECTED_URL)) {
throw new Error(
inputFile +

View File

@@ -3,10 +3,10 @@
# This script assumes ../curl/ is a git repo containing curl's source code
# and extracts the list of arguments curl accepts and writes the result as
# two JS objects (one for --long-options and one for -s (short) options)
# to util.js.
# to ../src/util.ts.
#
# curl defines its arguments in src/tool_getparam.c:
# https://github.com/curl/curl/blob/master/src/tool_getparam.c#L73
# https://github.com/curl/curl/blob/master/src/tool_getparam.c#L72
#
# Each argument definition is composed of
# letter - a 1 or 2 character string which acts as both a unique identifier
@@ -40,15 +40,15 @@ NEW_INPUT_FILE = CURL_REPO / "src" / "tool_getparam.c"
FILE_MOVED_TAG = "curl-7_23_0" # when the above change happened
# Originally there were only two arg "types": TRUE/FALSE which signified
# whether the option expected a value or was a boolean (respectively).
# whether the option expected a value or was a boolean, respectively.
# Then in
# 5abfdc0140df0977b02506d16796f616158bfe88
# which was released as
NO_OPTIONS_TAG = "curl-7_19_0"
# all boolean (i.e. FALSE "type") options got an implicit --no-OPTION.
# Then TRUE/FALSE was changed to ARG_STRING/ARG_BOOL.
# Then it was realized that not all options should have a --no-OPTION
# counterpart, so a new ARG_NONE type was added for those in
# Then it was realized that not all boolean options should have a
# --no-OPTION counterpart, so a new ARG_NONE type was added for those in
# 913c3c8f5476bd7bc4d8d00509396bd4b525b8fc
OPTS_START = "struct LongShort aliases[]= {"
@@ -59,14 +59,13 @@ STR_TYPES = ["string", "filename"]
ALIAS_TYPES = BOOL_TYPES + STR_TYPES
RAW_ALIAS_TYPES = ALIAS_TYPES + ["true", "false"]
OUTPUT_FILE = Path(__file__).parent / "util.js"
OUTPUT_FILE = Path(__file__).parent.parent / "src" / "util.ts"
JS_PARAMS_START = "BEGIN GENERATED CURL OPTIONS"
JS_PARAMS_END = "END GENERATED CURL OPTIONS"
PACKAGE_JSON = Path(__file__).parent / "package.json"
CLI_FILE = Path(__file__).parent / "bin" / "cli.js"
PACKAGE_JSON = Path(__file__).parent.parent / "package.json"
CLI_FILE = Path(__file__).parent.parent / "src" / "cli.ts"
CLI_VERSION_LINE_START = "const VERSION = "
# These are options with the same `letter`, which are options that were
@@ -171,7 +170,7 @@ def parse_aliases(lines):
return list(aliases.values())
def fill_out_aliases(aliases, add_no_options=True):
def fill_out_aliases(aliases, add_no_options=True, assumptions=set()):
# If both --option and --other-option have "oO" (for example) as their `letter`,
# add a "name" property with the main option's `lname`
letter_count = Counter(a["letter"] for a in aliases)
@@ -188,7 +187,10 @@ def fill_out_aliases(aliases, add_no_options=True):
if alias["type"] in BOOL_TYPES:
without_no = alias["lname"].removeprefix("no-").removeprefix("disable-")
if alias["lname"] != without_no:
print(f"Assuming --{alias['lname']} is {without_no!r}", file=sys.stderr)
assumption = f"Assuming --{alias['lname']} is {without_no!r}"
if assumption not in assumptions:
assumptions.add(assumption)
print(assumption, file=sys.stderr)
alias["name"] = without_no
if letter_count[alias["letter"]] > 1:
@@ -459,14 +461,16 @@ if __name__ == "__main__":
break
new_cli_lines.append(line)
else:
raise ValueError(f"no line in {CLI_FILE} starts with {CLI_VERSION_LINE_START!r}")
raise ValueError(
f"no line in {CLI_FILE} starts with {CLI_VERSION_LINE_START!r}"
)
new_cli_lines.append(cli_version_line)
for line in f:
new_cli_lines.append(line)
with open(OUTPUT_FILE, "w", newline="\n") as f:
f.write("".join(new_lines))
with open(CLI_FILE, "w", newline="\n") as f:
f.write("".join(new_cli_lines))
# with open(OUTPUT_FILE, "w", newline="\n") as f:
# f.write("".join(new_lines))
# with open(CLI_FILE, "w", newline="\n") as f:
# f.write("".join(new_cli_lines))