using System; using System.Collections; using System.Net; using System.Net.Http; using System.Text; using System.Text.Json; using System.Threading.Tasks; using Scriban.Functions; using Scriban.Helpers; using Scriban.Runtime; namespace Kalk.Core.Modules { /// /// Module that provides Web functions (e.g `url_encode`, `json`, `wget`...) /// [KalkExportModule(ModuleName)] public sealed partial class WebModule : KalkModuleWithFunctions { private const string ModuleName = "Web"; public const string CategoryWeb = "Web & Html Functions"; public WebModule() : base(ModuleName) { RegisterFunctionsAuto(); } /// /// Converts a specified URL text into a URL-encoded. /// /// URL encoding converts characters that are not allowed in a URL into character-entity equivalents. /// For example, when the characters < and > are embedded in a block of text to be transmitted in a URL, they are encoded as %3c and %3e. /// /// The url text to encode as an URL. /// An encoded URL. /// /// ```kalk /// >>> url_encode "this<is>an:url and another part" /// # url_encode("this<is>an:url and another part") /// out = "this%3Cis%3Ean%3Aurl+and+another+part" /// ``` /// [KalkExport("url_encode", CategoryWeb)] public string UrlEncode(string url) => WebUtility.UrlEncode(url); /// /// Converts a URL-encoded string into a decoded string. /// /// The URL to decode. /// The decoded URL /// /// ```kalk /// >>> url_decode "this%3Cis%3Ean%3Aurl+and+another+part" /// # url_decode("this%3Cis%3Ean%3Aurl+and+another+part") /// out = "this<is>an:url and another part" /// ``` /// [KalkExport("url_decode", CategoryWeb)] public string UrlDecode(string url) => WebUtility.UrlDecode(url); /// /// Identifies all characters in a string that are not allowed in URLS, and replaces the characters with their escaped variants. /// /// The input string. /// The input string url escaped /// /// ```kalk /// >>> "<hello> & <scriban>" |> url_escape /// # "<hello> & <scriban>" |> url_escape /// out = "%3Chello%3E%20&%20%3Cscriban%3E" /// ``` /// [KalkExport("url_escape", CategoryWeb)] public string UrlEscape(string url) => HtmlFunctions.UrlEscape(url); /// /// Encodes a HTML input string (replacing `&` by `&amp;`) /// /// The input string /// The input string with HTML entities. /// /// ```kalk /// >>> "<p>This is a paragraph</p>" |> html_encode /// # "<p>This is a paragraph</p>" |> html_encode /// out = "&lt;p&gt;This is a paragraph&lt;/p&gt;" /// >>> out |> html_decode /// # out |> html_decode /// out = "<p>This is a paragraph</p>" /// ``` /// [KalkExport("html_encode", CategoryWeb)] public string HtmlEncode(string text) => HtmlFunctions.Escape(text); /// /// Decodes a HTML input string (replacing `&amp;` by `&`) /// /// The input string /// The input string removed with any HTML entities. /// /// ```kalk /// >>> "<p>This is a paragraph</p>" |> html_encode /// # "<p>This is a paragraph</p>" |> html_encode /// out = "&lt;p&gt;This is a paragraph&lt;/p&gt;" /// >>> out |> html_decode /// # out |> html_decode /// out = "<p>This is a paragraph</p>" /// ``` /// [KalkExport("html_decode", CategoryWeb)] public string HtmlDecode(string text) { return string.IsNullOrEmpty(text) ? text : System.Net.WebUtility.HtmlDecode(text); } /// /// Converts to or from a JSON object depending on the value argument. /// /// A value argument: /// - If the value is a string, it is expecting this string to be a JSON string and will convert it to the appropriate object. /// - If the value is an array or object, it will convert it to a JSON string representation. /// /// A JSON string or an object/array depending on the argument. /// /// ```kalk /// >>> json {a: 1, b: 2, c: [4,5], d: "Hello World"} /// # json({a: 1, b: 2, c: [4,5], d: "Hello World"}) /// out = "{\"a\": 1, \"b\": 2, \"c\": [4, 5], \"d\": \"Hello World\"}" /// >>> json out /// # json(out) /// out = {a: 1, b: 2, c: [4, 5], d: "Hello World"} /// ``` /// [KalkExport("json", CategoryWeb)] public object Json(object value) { switch (value) { case string text: try { var jsonDoc = JsonDocument.Parse(text); return ConvertFromJson(jsonDoc.RootElement); } catch (Exception ex) { throw new ArgumentException($"Unable to parse input text. Reason: {ex.Message}", nameof(value)); } default: { var previousLimit = Engine.LimitToString; try { Engine.LimitToString = 0; var builder = new StringBuilder(); ConvertToJson(value, builder); return builder.ToString(); } catch (Exception ex) { throw new ArgumentException($"Unable to convert script object input to json text. Reason: {ex.Message}", nameof(value)); } finally { Engine.LimitToString = previousLimit; } } } } /// Removes any HTML tags from the input string /// The input string /// The input string removed with any HTML tags /// /// ```kalk /// >>> "<p>This is a paragraph</p>" |> html_strip /// # "<p>This is a paragraph</p>" |> html_strip /// out = "This is a paragraph" /// ``` /// [KalkExport("html_strip", CategoryWeb)] public string HtmlStrip(string text) => HtmlFunctions.Strip(Engine, text); /// /// Retrieves the content of the following URL by issuing a HTTP GET request. /// /// The URL to retrieve the content for. /// An object with the result of the request. This object contains the following members: /// - `version`: the protocol of the version. /// - `code`: the HTTP return code. /// - `reason`: the HTTP reason phrase. /// - `headers`: the HTTP returned headers. /// - `content`: the HTTP content. Either a string if the mime type is `text/*` or an object if the mime type is `application/json` otherwise it will return a bytebuffer. /// /// /// ``` /// >>> wget "https://markdig.azurewebsites.net/" /// # wget("https://markdig.azurewebsites.net/") /// out = {version: "1.1", code: 200, reason: "OK", headers: {"Content-Type": "text/plain; charset=utf-8", "Content-Length": 0}, content: ""} /// ``` /// [KalkExport("wget", CategoryWeb)] public ScriptObject WebGet(string url) { if (url == null) throw new ArgumentNullException(nameof(url)); using (var httpClient = new HttpClient()) { HttpResponseMessage result; try { var task = Task.Run(async () => await httpClient.GetAsync(url)); task.Wait(); result = task.Result; if (!result.IsSuccessStatusCode) { throw new ArgumentException($"HTTP/{result.Version} {(int)result.StatusCode} {result.ReasonPhrase}", nameof(url)); } else { var headers = new ScriptObject(); foreach (var headerItem in result.Content.Headers) { var items = new ScriptArray(headerItem.Value); object itemValue = items; if (items.Count == 1) { var str = (string) items[0]; itemValue = str; if (str == "true") { itemValue = (KalkBool)true; } else if (str == "false") { itemValue = (KalkBool)false; } else if (long.TryParse(str, out var longValue)) { if (longValue >= int.MinValue && longValue <= int.MaxValue) { itemValue = (int) longValue; } else { itemValue = longValue; } } else if (DateTime.TryParse(str, out var time)) { itemValue = time; } } headers[headerItem.Key] = itemValue; } var mediaType = result.Content.Headers.ContentType.MediaType; var resultObj = new ScriptObject() { {"version", result.Version.ToString()}, {"code", (int) result.StatusCode}, {"reason", result.ReasonPhrase}, {"headers", headers} }; if (mediaType.StartsWith("text/") || mediaType == "application/json") { var readTask = Task.Run(async () => await result.Content.ReadAsStringAsync()); readTask.Wait(); var text = readTask.Result; if (mediaType == "application/json" && TryParseJson(text, out var jsonObject)) { resultObj["content"] = jsonObject; } else { resultObj["content"] = readTask.Result; } } else { var readTask = Task.Run(async () => await result.Content.ReadAsByteArrayAsync()); readTask.Wait(); resultObj["content"] = new KalkNativeBuffer(readTask.Result); } return resultObj; } } catch (Exception ex) { throw new ArgumentException(ex.Message, nameof(url)); } } } private static bool TryParseJson(string text, out ScriptObject scriptObj) { scriptObj = null; try { // Make sure that we can parse the input JSON var jsonDoc = JsonDocument.Parse(text); var result = ConvertFromJson(jsonDoc.RootElement); scriptObj = result as ScriptObject; return scriptObj != null; } catch { // ignore return false; } } private void ConvertToJson(object element, StringBuilder builder) { if (element == null) { builder.Append("null"); } else if (element is ScriptObject scriptObject) { builder.Append("{"); bool isFirst = true; foreach (var item in scriptObject) { if (!isFirst) { builder.Append(", "); } builder.Append('\"'); builder.Append(StringFunctions.Escape(item.Key)); builder.Append('\"'); builder.Append(": "); ConvertToJson(item.Value, builder); isFirst = false; } builder.Append("}"); } else if (element is string text) { builder.Append('\"'); builder.Append(StringFunctions.Escape(text)); builder.Append('\"'); } else if (element.GetType().IsNumber()) { builder.Append(Engine.ObjectToString(element)); } else if (element is bool rb) { builder.Append(rb ? "true" : "false"); } else if (element is KalkBool b) { builder.Append(b ? "true" : "false"); } else if (element is IEnumerable it) { builder.Append('['); bool isFirst = true; foreach (var item in it) { if (!isFirst) { builder.Append(", "); } ConvertToJson(item, builder); isFirst = false; } builder.Append(']'); } else { builder.Append('\"'); builder.Append(StringFunctions.Escape(Engine.ObjectToString(element))); builder.Append('\"'); } } private static object ConvertFromJson(JsonElement element) { switch (element.ValueKind) { case JsonValueKind.Object: var obj = new ScriptObject(); foreach (var prop in element.EnumerateObject()) { obj[prop.Name] = ConvertFromJson(prop.Value); } return obj; case JsonValueKind.Array: var array = new ScriptArray(); foreach (var nestedElement in element.EnumerateArray()) { array.Add(ConvertFromJson(nestedElement)); } return array; case JsonValueKind.String: return element.GetString(); case JsonValueKind.Number: if (element.TryGetInt32(out var intValue)) { return intValue; } else if (element.TryGetInt64(out var longValue)) { return longValue; } else if (element.TryGetUInt32(out var uintValue)) { return uintValue; } else if (element.TryGetUInt64(out var ulongValue)) { return ulongValue; } else if (element.TryGetDecimal(out var decimalValue)) { return decimalValue; } else if (element.TryGetDouble(out var doubleValue)) { return doubleValue; } else { throw new InvalidOperationException($"Unable to convert number {element}"); } case JsonValueKind.True: return (KalkBool)true; case JsonValueKind.False: return (KalkBool)false; default: return null; } } } }