alihan/curlconverter
1
0
Fork 0
mirror of https://github.com/curlconverter/curlconverter.git synced 2022-05-22 02:35:29 +03:00
Code Issues Projects Releases Wiki Activity

add deleted curl arguments (#280)

Browse Source 
from curl's git repo's history
This commit is contained in:
Борис Верховский
2021-10-02 11:34:20 -06:00
committed by GitHub
parent a12a753f21
commit 8f414fc09f
6 changed files with 285 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@@ -1,17 +1,17 @@
# curlconverter
`curlconverter` transpiles [`curl`](https://en.wikipedia.org/wiki/CURL) Bash commands into programs in other programming languages.
`curlconverter` transpiles [`curl`](https://en.wikipedia.org/wiki/CURL) commands into programs in other programming languages.
```sh
$ curlconverter -X PUT --data "Hello, world!" example.com
$ curlconverter --data "Hello, world!" example.com
import requests
data = 'Hello, world!'
response = requests.put('http://example.com', data=data)
response = requests.post('http://example.com', data=data)
```
You can choose the output language by passing `--language <language>`. The options are Python `python` (the default), JavaScript `browser` `node` `node-request`, Go `go`, Rust `rust`, PHP `php`, Java `java`, R `r`, Elixir `elixir`, Dart `dart`, MATLAB `matlab` and a couple more.
You can choose the output language by passing `--language <language>`. The options are Python `python` (the default), JavaScript `browser` `node` `node-request`, Go `go`, Rust `rust`, PHP `php`, Java `java`, R `r`, Elixir `elixir`, Dart `dart`, MATLAB `matlab` and a few more.
[![NPM version][npm-image]][npm-url]
@@ -37,7 +37,7 @@ curlconverter requires Node 12+.
## Usage
The JavaScript API is a bunch of functions that can take either a string or an array
The JavaScript API is a bunch of functions that can take either a string of Bash code or an array
```js
import * as curlconverter from 'curlconverter';

3
bin/cli.js
View File

@@ -20,7 +20,8 @@ import { _toStrest } from '../generators/strest.js'
import fs from 'fs'
const VERSION = '4.0.0-alpha.6 (curl 7.79.1)'
// This line is generated by extract_curl_args.py. Do not modify it.
const VERSION = '4.0.0-alpha.7 (curl 7.79.1)'
// sets a default in case --language isn't passed
const defaultLanguage = 'python'

216
extract_curl_args.py
View File

@@ -25,16 +25,31 @@
# the old name needs to also be kept for backwards compatibility. To these
# options we add a "name" property with the newest name.
from pathlib import Path
import sys
import json
import subprocess
import sys
from collections import Counter
from pathlib import Path
# Git repo of curl's source code to extract the args from
# TODO: make this a command line arg?
# TODO: make this an optional command line arg
CURL_REPO = Path(__file__).parent.parent / "curl"
INPUT_FILE = CURL_REPO / "src" / "tool_getparam.c"
OUTPUT_FILE = Path(__file__).parent / "util.js"
OLD_INPUT_FILE = CURL_REPO / "src" / "main.c"
NEW_INPUT_FILE = CURL_REPO / "src" / "tool_getparam.c"
FILE_MOVED_TAG = "curl-7_23_0" # when the above change happened
# Originally there were only two arg "types": TRUE/FALSE which signified
# whether the option expected a value or was a boolean (respectively).
# Then in
# 5abfdc0140df0977b02506d16796f616158bfe88
# which was released as
NO_OPTIONS_TAG = "curl-7_19_0"
# all boolean (i.e. FALSE "type") options got an implicit --no-OPTION.
# Then TRUE/FALSE was changed to ARG_STRING/ARG_BOOL.
# Then it was realized that not all options should have a --no-OPTION
# counterpart, so a new ARG_NONE type was added for those in
# 913c3c8f5476bd7bc4d8d00509396bd4b525b8fc
OPTS_START = "struct LongShort aliases[]= {"
OPTS_END = "};"
@@ -42,10 +57,18 @@ OPTS_END = "};"
BOOL_TYPES = ["bool", "none"]
STR_TYPES = ["string", "filename"]
ALIAS_TYPES = BOOL_TYPES + STR_TYPES
RAW_ALIAS_TYPES = ALIAS_TYPES + ["true", "false"]
OUTPUT_FILE = Path(__file__).parent / "util.js"
JS_PARAMS_START = "BEGIN GENERATED CURL OPTIONS"
JS_PARAMS_END = "END GENERATED CURL OPTIONS"
PACKAGE_JSON = Path(__file__).parent / "package.json"
CLI_FILE = Path(__file__).parent / "bin" / "cli.js"
CLI_VERSION_LINE_START = "const VERSION = "
# These are options with the same `letter`, which are options that were
# renamed, along with their new name.
DUPES = {
@@ -57,6 +80,16 @@ DUPES = {
"ssl-reqd": "ssl-reqd",
"proxy-service-name": "proxy-service-name",
"socks5-gssapi-service": "proxy-service-name",
# These argument names have been deleted,
# they should appear as deleted options.
"request": "request",
"http-request": "request",
"use-ascii": "use-ascii",
"ftp-ascii": "use-ascii",
"ftpport": "ftp-port",
"ftp-port": "ftp-port",
"socks": "socks5",
"socks5": "socks5",
}
if not OUTPUT_FILE.is_file():
@@ -83,6 +116,16 @@ def git_branch(git_dir=CURL_REPO):
return branch.strip()
def is_git_repo(git_dir=CURL_REPO):
result = subprocess.run(
["git", "rev-parse", "--is-inside-work-tree"],
cwd=git_dir,
capture_output=True,
text=True,
)
return result.returncode == 0 and result.stdout.strip() == "true"
def parse_aliases(lines):
aliases = {}
for line in lines:
@@ -112,7 +155,7 @@ def parse_aliases(lines):
if 1 > len(letter) > 2:
raise ValueError(f"letter form of --{lname} must be 1 or 2 characters long")
if type_ not in ALIAS_TYPES:
if type_ not in RAW_ALIAS_TYPES:
raise ValueError(f"unknown desc for --{lname}: {desc!r}")
alias = {"letter": letter, "lname": lname, "type": type_}
@@ -137,9 +180,9 @@ def fill_out_aliases(aliases, add_no_options=True):
no_aliases = []
for idx, alias in enumerate(aliases):
if alias["type"] == "false":
alias["type"] = "string"
if alias["type"] == "true":
alias["type"] = "string"
if alias["type"] == "false":
alias["type"] = "bool" if add_no_options else "none"
if alias["type"] in BOOL_TYPES:
@@ -198,8 +241,8 @@ def split(aliases):
return long_args, short_args
def format_as_js(d, var_name, indent=0, indent_type='\t'):
yield f"{indent_type * indent}var {var_name} = {{"
def format_as_js(d, var_name, indent="\t", indent_level=0):
yield f"{indent * indent_level}const {var_name} = {{"
for top_key, opt in d.items():
def quote(key):
@@ -214,14 +257,14 @@ def format_as_js(d, var_name, indent=0, indent_type='\t'):
if isinstance(opt, dict):
vals = [f"{quote(k)}: {val_to_js(v)}" for k, v in opt.items()]
yield f"{indent_type * (indent + 1)}{top_key!r}: {{{', '.join(vals)}}},"
yield f"{indent * (indent_level + 1)}{top_key!r}: {{{', '.join(vals)}}},"
elif isinstance(opt, str):
yield f"{indent_type * (indent + 1)}{top_key!r}: {val_to_js(opt)},"
yield f"{indent * (indent_level + 1)}{top_key!r}: {val_to_js(opt)},"
yield (indent_type * indent) + "};"
yield (indent * indent_level) + "};"
def parse_version(tag):
def parse_tag(tag):
if not tag.startswith("curl-") or tag.startswith("curl_"):
return None
version = tag.removeprefix("curl-").removeprefix("curl_")
@@ -254,9 +297,8 @@ def curl_tags(git_dir=CURL_REPO):
.splitlines()
)
for tag in tags:
version = parse_version(tag)
if version:
yield tag, version
if parse_tag(tag):
yield tag
def file_at_commit(filename, commit_hash, git_dir=CURL_REPO):
@@ -274,24 +316,117 @@ def file_at_commit(filename, commit_hash, git_dir=CURL_REPO):
if __name__ == "__main__":
if git_branch(CURL_REPO) != "master":
sys.exit("not on curl repo's git master")
# TODO: check that repo is up to date
if not is_git_repo(CURL_REPO):
sys.exit(f"{CURL_REPO} is not a git repo")
*tags, last_tag = sorted(curl_tags(CURL_REPO), key=lambda x: x[1])
tags = sorted(curl_tags(CURL_REPO), key=parse_tag)
aliases = fill_out_aliases(
parse_aliases(file_at_commit("src/tool_getparam.c", last_tag[0]))
)
long_args, short_args = split(aliases)
moved_file = False
aliases = {}
short_aliases = {}
filename = "src/main.c"
add_no_options = False
# for tag, version in
# old_aliases = fill_out_aliases(parse_aliases(f), add_no_options)
for tag in tags:
if tag == FILE_MOVED_TAG:
filename = "src/tool_getparam.c"
if tag == NO_OPTIONS_TAG:
add_no_options = True
f = file_at_commit(filename, tag)
aliases[tag] = {}
short_aliases[tag] = {}
for alias in fill_out_aliases(parse_aliases(f), add_no_options):
alias["expand"] = alias.get("expand", True)
alias_name = alias.get("name", alias["lname"])
alias_name = DUPES.get(alias_name, alias_name)
# alias['name'] = alias_name
if alias["lname"] in aliases[tag] and aliases[tag][alias["lname"]] != alias:
raise ValueError("duplicate alias: --" + alias["lname"])
# We don't want to report when curl changed the internal ID of some option
if len(alias["letter"]) == 1:
short_aliases[tag][alias["letter"]] = (alias_name, alias["type"])
del alias["letter"]
lname = alias["lname"]
del alias["lname"]
# TODO: figure out what to do about how shortenings change over time
del alias["expand"]
aliases[tag][lname] = alias
# TODO: report how shortened --long options change
js_params_lines = list(format_as_js(long_args, "curlLongOpts", indent_type=" "))
for cur_tag, next_tag in zip(aliases.keys(), list(aliases.keys())[1:]):
cur_aliases = short_aliases[cur_tag]
next_aliases = short_aliases[next_tag]
latest_aliases = short_aliases[list(aliases.keys())[-1]]
# We don't care about when options got added
# new_aliases = next_aliases.keys() - cur_aliases.keys()
removed_aliases = cur_aliases.keys() - next_aliases.keys()
changed_aliases = []
for common_alias in cur_aliases.keys() & next_aliases.keys():
if cur_aliases[common_alias] != next_aliases[common_alias]:
changed_aliases.append(common_alias)
if removed_aliases or changed_aliases:
header = f"{cur_tag} -> {next_tag}"
print(header)
print("=" * len(header))
for removed_alias in removed_aliases:
print(f"- -{removed_alias} {cur_aliases[removed_alias]}")
currently = latest_aliases.get(removed_alias)
if currently:
# Could've been removed and added back multiple times, so what
# it is on master is not necessarily how it was added back next.
print(" added back later and is currently " + str(currently))
print()
for changed_alias in changed_aliases:
print(f"- -{changed_alias} {cur_aliases[changed_alias]}")
print(f"+ -{changed_alias} {next_aliases[changed_alias]}")
currently = latest_aliases.get(changed_alias, "(no longer exists)")
if currently != next_aliases[changed_alias]:
print(" later became " + str(currently))
print()
print("-" * 80)
print()
for cur_tag, next_tag in zip(aliases.keys(), list(aliases.keys())[1:]):
cur_aliases = aliases[cur_tag]
next_aliases = aliases[next_tag]
new_aliases = next_aliases.keys() - cur_aliases.keys()
removed_aliases = cur_aliases.keys() - next_aliases.keys()
changed_aliases = []
for common_alias in cur_aliases.keys() & next_aliases.keys():
if cur_aliases[common_alias] != next_aliases[common_alias]:
changed_aliases.append(common_alias)
# We don't care when aliases were added, only when/if they are removed,
# but we need to be able to see if an alias was added because it's actually
# replacing a previous alias.
# Only reporting added aliases when there are removed or changed aliases
# is probably good enough for that purpose.
if removed_aliases or changed_aliases: # or new_aliases:
header = f"{cur_tag} -> {next_tag}"
print(header)
print("=" * len(header))
for new_alias in new_aliases:
print(f"+ --{new_alias}: {next_aliases[new_alias]}")
print()
for removed_alias in removed_aliases:
print(f"- --{removed_alias}: {cur_aliases[removed_alias]}")
print()
for changed_alias in changed_aliases:
print(f"- --{changed_alias}: {cur_aliases[changed_alias]}")
print(f"+ --{changed_alias}: {next_aliases[changed_alias]}")
print()
current_aliases = fill_out_aliases(
parse_aliases(file_at_commit(filename, tags[-1])), add_no_options
)
long_args, short_args = split(current_aliases)
js_params_lines = list(format_as_js(long_args, "curlLongOpts", indent=" "))
js_params_lines += [""] # separate by a newline
js_params_lines += list(format_as_js(short_args, "curlShortOpts", indent_type=" "))
js_params_lines += list(format_as_js(short_args, "curlShortOpts", indent=" "))
new_lines = []
with open(OUTPUT_FILE) as f:
@@ -312,5 +447,26 @@ if __name__ == "__main__":
for line in f:
new_lines.append(line)
new_cli_lines = []
curl_version = tags[-1].removeprefix("curl-").replace("_", ".")
with open(PACKAGE_JSON) as f:
package_version = json.load(f)["version"]
cli_version = f"{package_version} (curl {curl_version})"
cli_version_line = CLI_VERSION_LINE_START + repr(cli_version) + "\n"
with open(CLI_FILE) as f:
for line in f:
if line.strip().startswith(CLI_VERSION_LINE_START):
break
new_cli_lines.append(line)
else:
raise ValueError(f"no line in {CLI_FILE} starts with {CLI_VERSION_LINE_START!r}")
new_cli_lines.append(cli_version_line)
for line in f:
new_cli_lines.append(line)
with open(OUTPUT_FILE, "w", newline="\n") as f:
f.write("".join(new_lines))
with open(CLI_FILE, "w", newline="\n") as f:
f.write("".join(new_cli_lines))

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "curlconverter",
"version": "4.0.0-alpha.6",
"version": "4.0.0-alpha.7",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "curlconverter",
"version": "4.0.0-alpha.6",
"version": "4.0.0-alpha.7",
"license": "MIT",
"dependencies": {
"cookie": "^0.4.1",

2
package.json
View File

@@ -1,6 +1,6 @@
{
"name": "curlconverter",
"version": "4.0.0-alpha.6",
"version": "4.0.0-alpha.7",
"description": "convert curl commands to Python, JavaScript, Go, PHP and other languages",
"homepage": "https://github.com/NickCarneiro/curlconverter",
"author": {

90
util.js
View File

@@ -442,11 +442,80 @@ const curlShortOpts = {
}
// END GENERATED CURL OPTIONS
// These are options that curl used to have.
// Those that don't conflict with the current options are supported by curlconverter.
// TODO: curl's --long-options can be shortened.
// For example if curl used to only have a single option, "--blah" then
// "--bla" "--bl" and "--b" all used to be valid options as well. If later
// "--blaz" was added, suddenly those 3 shortened options are removed (because
// they are now ambiguous).
// https://github.com/NickCarneiro/curlconverter/pull/280#issuecomment-931241328
const removedLongOpts = {
'ftp-ascii': { type: 'bool', name: 'use-ascii', removed: '7.10.7' },
port: { type: 'string', removed: '7.3' },
upload: { type: 'bool', removed: '7.7' },
continue: { type: 'bool', removed: '7.9' },
'3p-url': { type: 'string', removed: '7.16.0' },
'3p-user': { type: 'string', removed: '7.16.0' },
'3p-quote': { type: 'string', removed: '7.16.0' },
'http2.0': { type: 'bool', name: 'http2', removed: '7.36.0' },
'no-http2.0': { type: 'bool', name: 'http2', removed: '7.36.0' },
'telnet-options': { type: 'string', name: 'telnet-option', removed: '7.49.0' },
'http-request': { type: 'string', name: 'request', removed: '7.49.0' },
socks: { type: 'string', name: 'socks5', removed: '7.49.0' },
'capath ': { type: 'string', name: 'capath', removed: '7.49.0' }, // trailing space
ftpport: { type: 'string', name: 'ftp-port', removed: '7.49.0' },
environment: { type: 'bool', removed: '7.54.1' },
// These --no-<option> flags were automatically generated and never had any effect
'no-tlsv1': { type: 'bool', name: 'tlsv1', removed: '7.54.1' },
'no-tlsv1.2': { type: 'bool', name: 'tlsv1.2', removed: '7.54.1' },
'no-http2-prior-knowledge': { type: 'bool', name: 'http2-prior-knowledge', removed: '7.54.1' },
'no-ipv6': { type: 'bool', name: 'ipv6', removed: '7.54.1' },
'no-ipv4': { type: 'bool', name: 'ipv4', removed: '7.54.1' },
'no-sslv2': { type: 'bool', name: 'sslv2', removed: '7.54.1' },
'no-tlsv1.0': { type: 'bool', name: 'tlsv1.0', removed: '7.54.1' },
'no-tlsv1.1': { type: 'bool', name: 'tlsv1.1', removed: '7.54.1' },
'no-remote-name': { type: 'bool', name: 'remote-name', removed: '7.54.1' },
'no-sslv3': { type: 'bool', name: 'sslv3', removed: '7.54.1' },
'no-get': { type: 'bool', name: 'get', removed: '7.54.1' },
'no-http1.0': { type: 'bool', name: 'http1.0', removed: '7.54.1' },
'no-next': { type: 'bool', name: 'next', removed: '7.54.1' },
'no-tlsv1.3': { type: 'bool', name: 'tlsv1.3', removed: '7.54.1' },
'no-environment': { type: 'bool', name: 'environment', removed: '7.54.1' },
'no-http1.1': { type: 'bool', name: 'http1.1', removed: '7.54.1' },
'no-proxy-tlsv1': { type: 'bool', name: 'proxy-tlsv1', removed: '7.54.1' },
'no-http2': { type: 'bool', name: 'http2', removed: '7.54.1' }
}
for (const [opt, val] of Object.entries(removedLongOpts)) {
if (!has(val, 'name')) {
val.name = opt
}
}
// TODO: use this to warn users when they specify a short option that
// used to be for something else?
const changedShortOpts = {
p: 'used to be short for --port <port> (a since-deleted flag) until curl 7.3',
// TODO: some of these might be renamed options
t: 'used to be short for --upload (a since-deleted boolean flag) until curl 7.7',
c: 'used to be short for --continue (a since-deleted boolean flag) until curl 7.9',
// TODO: did -@ actually work?
'@': 'used to be short for --create-dirs until curl 7.10.7',
Z: 'used to be short for --max-redirs <num> until curl 7.10.7',
9: 'used to be short for --crlf until curl 7.10.8',
8: 'used to be short for --stderr <file> until curl 7.10.8',
7: 'used to be short for --interface <name> until curl 7.10.8',
6: 'used to be short for --krb <level> (which itself used to be --krb4 <level>) until curl 7.10.8',
// TODO: did these short options ever actually work?
5: 'used to be another way to specify the url until curl 7.10.8',
'*': 'used to be another way to specify the url until curl 7.49.0',
'~': 'used to be short for --xattr until curl 7.49.0'
}
// These options can be specified more than once, they
// are always returned as a list.
// Normally, if you specify some option more than once,
// curl will just take the last one.
// TODO: extract this from curl's source code
// TODO: extract this from curl's source code?
const canBeList = new Set([
// TODO: unlike curl, we don't support multiple
// URLs and just take the last one.
@@ -492,6 +561,22 @@ for (const [shortenedOpt, vals] of Object.entries(shortened)) {
}
}
}
for (const [removedOpt, val] of Object.entries(removedLongOpts)) {
if (!has(curlLongOpts, removedOpt)) {
curlLongOpts[removedOpt] = val
} else if (curlLongOpts[removedOpt] === null) {
// This happens with --socks because it became --socks5 and there are multiple options
// that start with "--socks"
// console.error("couldn't add removed option --" + removedOpt + " to curlLongOpts because it's already ambiguous")
// TODO: do we want to do this?
// curlLongOpts[removedOpt] = val
} else {
// Almost certainly a shortened form of a still-existing option
// This happens with --continue (now short for --continue-at)
// and --upload (now short for --upload-file)
// console.error("couldn't add removed option --" + removedOpt + ' to curlLongOpts because it already exists')
}
}
const toBoolean = opt => {
if (opt.startsWith('no-disable-')) {
@@ -711,6 +796,9 @@ const parseArgs = (args, opts) => {
}
for (let j = 1; j < arg.length; j++) {
if (!has(shortOpts, arg[j])) {
if (has(changedShortOpts, arg[j])) {
throw 'option ' + arg + ': ' + changedShortOpts[arg[j]]
}
// TODO: there are a few deleted short options we could report
throw 'option ' + arg + ': is unknown'
}