From 9e5dafacf12f4066397f7b5ddd95acf192109df1 Mon Sep 17 00:00:00 2001 From: bear8642 Date: Mon, 22 Dec 2025 13:33:25 +0000 Subject: [PATCH 01/37] Start adding application tuning guide --- application-tuning-guide/docs/index.md | 15 ++++++ application-tuning-guide/docs/introduction.md | 29 +++++++++++ application-tuning-guide/mkdocs.yml | 49 +++++++++++++++++++ application-tuning-guide/print_mkdocs.yml | 47 ++++++++++++++++++ 4 files changed, 140 insertions(+) create mode 100644 application-tuning-guide/docs/index.md create mode 100644 application-tuning-guide/docs/introduction.md create mode 100644 application-tuning-guide/mkdocs.yml create mode 100644 application-tuning-guide/print_mkdocs.yml diff --git a/application-tuning-guide/docs/index.md b/application-tuning-guide/docs/index.md new file mode 100644 index 0000000000..925a17dd14 --- /dev/null +++ b/application-tuning-guide/docs/index.md @@ -0,0 +1,15 @@ +--- +search: + exclude: true +--- + +# About + +The _Application Tuning Guide_ describes the way in which the `⎕PROFILE` +system function and the associated `]PROFILE` user command can be +used to obtain a performance profile of an application. It describes +both the graphical and textual output that can be obtained following +data collection and shows how this data can be analysed and potential +inefficiencies identified. + +Assumed knowledge: A reasonable understanding of Dyalog APL. diff --git a/application-tuning-guide/docs/introduction.md b/application-tuning-guide/docs/introduction.md new file mode 100644 index 0000000000..dc4e6bbf38 --- /dev/null +++ b/application-tuning-guide/docs/introduction.md @@ -0,0 +1,29 @@ +

Introduction

+ +Application design includes assumptions about usage patterns and data +volumes. Over time, these can evolve to the detriment of the application's +performance. The most effective way to counter drops in performance caused +by changes external to the application is to identify the hot spots +(places in the application where a high proportion of CPU or Elapsed +Time is consumed); these hot spots can then be tuned to improve the +application's performance. + +The `⎕PROFILE` system function and the `]Profile` user command +facilitate the hot spot identification process; the `⎕PROFILE` system +function gathers statistics from an application and the `]Profile` +user command summarises, filters and reports on this data, simplifying +the process of drilling down on the (frequently large amounts of) data +returned by `⎕PROFILE`. + +!!! Legacy "Legacy " + The `⎕MONITOR` system function, which was in use prior to + the introduction of the `⎕PROFILE` system function, has been + deprecated and Dyalog Ltd recommends rewriting tools to use the + `⎕PROFILE` system function instead; `⎕PROFILE` provides high + precision timing, calling tree analysis, and superior dfn and + recursive code handling. + +For more information on the `⎕PROFILE` system function, see the Dyalog +APL Language Reference Guide. For more information on the `]Profile` +user command, see Appendix A. + diff --git a/application-tuning-guide/mkdocs.yml b/application-tuning-guide/mkdocs.yml new file mode 100644 index 0000000000..79275d85a5 --- /dev/null +++ b/application-tuning-guide/mkdocs.yml @@ -0,0 +1,49 @@ +site_name: UNIX Installation and Configuration Guide +theme: + favicon: documentation-assets/images/favicon-32.png + logo: documentation-assets/images/dyalog-logo_white.svg + features: + - navigation.instant + - navigation.tracking + - navigation.path + - navigation.indexes + name: material + font: + text: Be Vietnam Pro +extra_css: + - documentation-assets/css/main.css + - documentation-assets/css/extra.css +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js +plugins: + - privacy + - search + - macros + - site-urls + - caption: + table: + enable: true + start_index: 1 + increment_index: 1 + position: bottom + reference_text: 'Table {index}' + caption_prefix: 'Table {index}:' +extra: + version_maj: 20 + version_majmin: 20.0 +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.keys + - pymdownx.superfences + - pymdownx.arithmatex: + generic: true + - attr_list + - abbr + - footnotes + - md_in_html + - markdown_tables_extended + - toc: + title: On this page +nav: diff --git a/application-tuning-guide/print_mkdocs.yml b/application-tuning-guide/print_mkdocs.yml new file mode 100644 index 0000000000..5448e40df6 --- /dev/null +++ b/application-tuning-guide/print_mkdocs.yml @@ -0,0 +1,47 @@ +site_name: UNIX Installation and Configuration Guide +theme: + favicon: documentation-assets/images/favicon-32.png + logo: documentation-assets/images/dyalog-logo_white.svg + features: + - navigation.instant + - navigation.tracking + - navigation.path + - navigation.indexes + name: material + font: + text: Be Vietnam Pro +extra_css: + - documentation-assets/css/main.css + - documentation-assets/css/extra.css +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js +plugins: + - privacy + - search + - macros + - site-urls + - caption: + table: + enable: true + start_index: 1 + increment_index: 1 + position: bottom +extra: + version_maj: 20 + version_majmin: 20.0 +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.keys + - pymdownx.superfences + - pymdownx.arithmatex: + generic: true + - attr_list + - abbr + - footnotes + - md_in_html + - markdown_tables_extended + - toc: + title: On this page +nav: From b264ccf8ceb9ba870fd506b7a74a00fb6dbbcb9a Mon Sep 17 00:00:00 2001 From: bear8642 Date: Mon, 22 Dec 2025 14:12:46 +0000 Subject: [PATCH 02/37] Add data collection section --- .../docs/collection/collecting-data.md | 28 +++++++++++++++++++ .../docs/collection/index.md | 6 ++++ .../initiating-the-collection-of-data.md | 13 +++++++++ .../stopping-the-collection-of-data.md | 6 ++++ .../docs/collection/timer-overhead.md | 9 ++++++ 5 files changed, 62 insertions(+) create mode 100644 application-tuning-guide/docs/collection/collecting-data.md create mode 100644 application-tuning-guide/docs/collection/index.md create mode 100644 application-tuning-guide/docs/collection/initiating-the-collection-of-data.md create mode 100644 application-tuning-guide/docs/collection/stopping-the-collection-of-data.md create mode 100644 application-tuning-guide/docs/collection/timer-overhead.md diff --git a/application-tuning-guide/docs/collection/collecting-data.md b/application-tuning-guide/docs/collection/collecting-data.md new file mode 100644 index 0000000000..709bb7ecf3 --- /dev/null +++ b/application-tuning-guide/docs/collection/collecting-data.md @@ -0,0 +1,28 @@ +## Collecting Data + +Whether data is being collected or not can be verified by entering: +```apl +⎕PROFILE 'state' +``` + +This returns a 4 element vector, in which: + +|-----|---------------------------| +|`[1]`|is a character vector indicating the state of `⎕PROFILE`.
Can be either `active` or `inactive` (must be `active` for data collection).| +|`[2]`|is a character vector indicating the timer being used.
Can be `CPU`, `elapsed`, `coverage` or `none`.| +|`[3]`| is the call time bias in milliseconds, that is the amount of time that is consumed when the system takes a time measurement.| +|`[4]`| is the timer granularity in milliseconds, that is, the resolution of the timer being used. On most platforms this will be zero, indicating that the granularity is smaller than the cost and cannot be estimated.| + +During data collection, the following data is recorded for each function and for each individual line in a function: + * Calls – the number of times the function or line was executed. + * Exclusive Time – milliseconds spent executing the function or line, excluding time spent in functions that were called by the function or line. + * Inclusive Time – milliseconds spent executing the function or line, including time spent in functions that were called by the function or line. + +The times collected by the `⎕PROFILE` system function include the time spent calling the timer function. This means that lines that are called a large number of times can appear to consume more resource than they actually do. For more accurate profiling measurements, adjustments should be made for the timer call time bias. To do this, the application should be run for a sufficiently long period to collect enough data to overcome the timer granularity – a reasonable rule of thumb is to make sure the application runs for at least `4000×4⊃⎕PROFILE 'state'` milliseconds. + +The profiling data that is collected is stored outside the workspace and does not impact workspace availability. + +`⎕PROFILE` can collect data for functions that are dynamically paged in and out of the workspace. + +!!! Info "Information" + Results can be confusing if several different functions with the same name are used at different times during execution – these are treated as the same function by `⎕PROFILE`. diff --git a/application-tuning-guide/docs/collection/index.md b/application-tuning-guide/docs/collection/index.md new file mode 100644 index 0000000000..398ba843da --- /dev/null +++ b/application-tuning-guide/docs/collection/index.md @@ -0,0 +1,6 @@ +

Data Collection

+ +!!! Warning "Warning" + The `⎕PROFILE` system function can collect very large quantities of data. This means that, to profile a large application or to save a dataset as an XML file, the workspace size might need to be increased significantly. + +For complete documentation of the `⎕PROFILE` system function, see the Dyalog APL Language Reference Guide. \ No newline at end of file diff --git a/application-tuning-guide/docs/collection/initiating-the-collection-of-data.md b/application-tuning-guide/docs/collection/initiating-the-collection-of-data.md new file mode 100644 index 0000000000..c0591a3b7b --- /dev/null +++ b/application-tuning-guide/docs/collection/initiating-the-collection-of-data.md @@ -0,0 +1,13 @@ +## Initiating the Collection of Data + +Data collection is initiated by entering: +```apl +⎕PROFILE'start' +``` + +This puts ⎕PROFILE into an active state. + +⎕PROFILE supports profiling using either CPU or elapsed time. CPU +time is usually of more interest in profiling application performance +and, by default, ⎕PROFILE will register CPU usage data using the +best available counter. diff --git a/application-tuning-guide/docs/collection/stopping-the-collection-of-data.md b/application-tuning-guide/docs/collection/stopping-the-collection-of-data.md new file mode 100644 index 0000000000..54df46fb12 --- /dev/null +++ b/application-tuning-guide/docs/collection/stopping-the-collection-of-data.md @@ -0,0 +1,6 @@ +## Stopping the Collection of Data +Data collection is stopped by entering: +```apl +⎕PROFILE 'stop' +``` +This puts `⎕PROFILE` into an inactive state. diff --git a/application-tuning-guide/docs/collection/timer-overhead.md b/application-tuning-guide/docs/collection/timer-overhead.md new file mode 100644 index 0000000000..2d4511572d --- /dev/null +++ b/application-tuning-guide/docs/collection/timer-overhead.md @@ -0,0 +1,9 @@ +## Timer Overhead + +As with all system timers, a cost is associated with the collection of timing data using the `⎕PROFILE` system function. In optimised applications the overhead can be significant; although it is unlikely to impact the identification of hot spots, it can distort results. + +By default, reports produced with the `]Profile` user command automatically adjust for timer bias, using the recorded bias – this can be disabled for a report by including the `‑bias=0` modifier and modifier value. + +The cost of querying a timer can vary significantly with system load, and repeatable timings are only possible if there is very little activity on the system; the variability is due to the timer calling the operating system kernel, which is servicing all processes on the machine. Dyalog Ltd. recommends increasing the priority of the application being profiled as this can reduce the variability (but will not eliminate it completely). + +The timer cost and granularity are estimated the first time that `⎕PROFILE` is run in an APL session. A new calibration can be requested by calling `⎕PROFILE 'calibrate'`. From a95dfc2003d632f48b84ec4c253c7578737f5d44 Mon Sep 17 00:00:00 2001 From: bear8642 Date: Mon, 22 Dec 2025 15:13:55 +0000 Subject: [PATCH 03/37] Add Appendix A --- ...pendix-a-syntax-of-profile-user-command.md | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 application-tuning-guide/docs/appendix-a-syntax-of-profile-user-command.md diff --git a/application-tuning-guide/docs/appendix-a-syntax-of-profile-user-command.md b/application-tuning-guide/docs/appendix-a-syntax-of-profile-user-command.md new file mode 100644 index 0000000000..9b0d32fc05 --- /dev/null +++ b/application-tuning-guide/docs/appendix-a-syntax-of-profile-user-command.md @@ -0,0 +1,72 @@ +

Appendix A

+ +Syntax of the `]Profile` User Command + +The `]Profile` user command is always followed by a report type; modifiers can be included to customise the output. + +Syntax: ]Profile [reporttype][-avg][-code][-lines]|[-outfile{=name}] [-format{=xml|csv|txt}][-cumpct][-exclusive][-first{=n}|-pct{=n}][-fn{=name}][-infile{=name}][-separators{="decimalsep phrasesep"}][-bias{=t}][-decimal{=n}][-expr{=expression}][-title{=name}] + +# Report Types + +The six possible report types are detailed below. If no report type is specified then a default report type is assumed; this is dashboard on the Microsoft Windows operating system and summary on the AIX, Linux and Mac OS operating systems. + +|Report Type|Description| +|`calls`|Shows how the consumption of a named function (the `-fn` modifier is required) is broken down by calling function.
The `summary` and `calls` report types are the most frequently used reporting tools.| +|`dashboard`|Opens the Dashboard, a graphical overview of the profiling data collected by the `⎕PROFILE` system function. For more information on the Dashboard, see Appendix B. +|`data`|Writes the raw data produced by `⎕PROFILE'data'` to a file for use with tools other than `]Profile`, for example, Microsoft Excel.| +|`state`|Displays the current profiling state of `⎕PROFILE` (see Section).| +|`summary`|Reports the number of calls, total consumption and consumption as a percentage of overall consumption.
The `summary` and `calls` report types are the most frequently used reporting tools.
This is the default report type on the AIX, Linux and macOS operating systems.| +|`tree`|Writes the raw data produced by `⎕PROFILE'tree'` to a file for later use. Intended as a tool for storing data using the `-outfile=` modifier, for subsequent reporting using the `-infile=` modifier.| + +# Modifiers + +The report types can be qualified using modifiers. These can, for example, filter the data that is displayed, add optional output columns, read input from a previously saved file or store the results of a command in a file. + +Each of the report types can have different combinations of modifiers applied. The state report type does not take any modifiers; the valid modifiers for each of the other report types are shown in. the following table: + +|Modifier |Report Types || +|_^ _|calls|dashboard|data|summary|tree| +-avg |y| |y|y| | +-bias |y|y|y|y|y| +-code |y| | | |y| +|-cumpct |y| |y| |y| +|-decimal |y| |y|y|y| +|-exclusive |y| |y| |y| +|-expr |y|y|y|y|y| +|-first |y| |y| |y| +|-fn |y|y|y|y| | +|-format |y| |y|y|y| +|-infile |y|y|y|y|y| +|-lines |y| |y|y|y| +|-outfile |y| |y|y|y| +|-pct |y| |y|y| | +|-separators|y *| |y *|y *|y *| +|-title** |y ** y **| y **| y **| y **| + +* can only be used when -“format=csv is included +** only relevant when -“format=xml or -“format=txt + +|Modifier| Description| +|-avg| Includes the average CPU consumption (in ms) per execution of each function call (or line if the -lines modifier is specified). +|-code| Includes the source code for the line being executed (including the -code modifier forces the -lines modifier).
Cannot be used with the -outfile modifier.| +|-cumpct|Displays the cumulative percentage of overall CPU consumption that each function call (or line if the -lines modifier is specified) and each function call above it was responsible for.
This is usually only useful if the -exclusive modifier is also set.| +|-exclusive|Displays the CPU consumption of each function call (or line if the -lines modifier is specified) excluding consumption due to called functions.| +|-first= n|After sorting into descending order of CPU consumption, displays only the first n function calls (or lines if the -lines modifier is specified).
This is usually only useful if the -exclusive modifier is also set.
Cannot be used with the -pct modifier. +|-fn= name|Mandatory for a calls report type, when it specifies the function that the calls analysis report is for. Optional for other report types, when output is filtered to only include data for the specified function and other functions that it calls. +|-lines|Displays a breakdown of consumption by individual line rather than a total for each function (the default).
Assumed when the -code modifier is specified.| +|-pct= n|After sorting into descending order of CPU consumption, displays only those function calls (or lines if the -lines modifier is specified) for which the cumulative percentage of overall CPU consumption is less than or equal to n.
This is usually only useful if the -exclusive modifier is also set.
Cannot be used with the -first modifier.| +|-format= n|Selects the file format to use when saving a file using the -outfile modifier. Possible values are: +* xml - writes data to an XML file (the default) +* csv - writes data to a CSV file +* txt - writes data to a text file (and retains the display format) +|-infile= n| Opens the Dashboard on the dataset contained in the specified.xml file.
Doing this does not destroy any existing `⎕PROFILE` data.| +|-separators= nn|For use with -format=csv.
Specifies the decimal and comma separators to use.
The default is '.,'| +|-title= n|For use with -format=xml or -format=txt.
Specifies the string that is used as a title caption in the Dashboard and XML reports. Especially useful when running the same expression multiple times as different captions can differentiate between different sets of results.
If the -title modifier is not specified, then the caption defaults to the string specified by the -expr modifier. If neither the -title nor the -expr modifiers are specified, then the caption defaults to `]profile Dashboard: