Skip to content

Commit

Permalink
+ `for await (const page_data of wiki.allpages({ namespace: 'Talk', a…
Browse files Browse the repository at this point in the history 
…pfrom: wiki.remove_namespace('ABC') })) {}`
  • Loading branch information
@kanasimi
kanasimi committed Jan 15, 2024
1 parent 02ec0a3 commit b65a3cd
Show file tree
Hide file tree
Showing 5 changed files with 244 additions and 40 deletions.
120 changes: 111 additions & 9 deletions Wikiapi.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ const wiki = new Wikiapi('en');
const page_data = await wiki.page('JavaScript');
const parsed = page_data.parse();
let infobox;
// Read Infobox templates, convert to JSON.
// Read [[w:en:MOS:INFOBOX|Infobox templates]], convert to JSON.
parsed.each('template', template_token => {
if (template_token.name.startsWith('Infobox')) {
infobox = template_token.parameters;
Expand Down Expand Up @@ -1017,6 +1017,7 @@ const wiki = new Wikiapi;
let list = await wiki.embeddedin('Template:Periodic table');
console.log(list);
// </code>
*
*/

// Warning: Won't throw if title is not existed!
Expand Down Expand Up @@ -1763,7 +1764,7 @@ function Wikiapi_get_featured_content(options) {
if (!options || !options.type) {
const session = this;
return Wikiapi_get_featured_content.default_types
.reduce((promise, type) => promise.then(Wikiapi_get_featured_content.bind(session, { ...options, type })), Promise.resolve());
.reduce((promise, type) => promise.then(Wikiapi_get_featured_content.bind(session, { ignore_missed: true, ...options, type, })), Promise.resolve());
if (false) {
let promise = Promise.resolve();
Wikiapi_get_featured_content.default_types.forEach(type => {
Expand Down Expand Up @@ -1967,8 +1968,35 @@ for (const property_name of ('task_configuration|latest_task_configuration').spl
});
}


/**
*
* @example <caption>Process each page of the category.</caption>
// <code>
const page_list = await wiki.categorymembers('Category:Articles not listed in the vital article list');
wiki.categorymembers('Category:Articles not listed in the vital article list', { for_each_page(page_data) { console.log('page_data:', page_data); } })
for await (const page_data of wiki.categorymembers('Category:Articles not listed in the vital article list')) {
console.trace('page_data:', page_data);
}
console.log('All done.');
// </code>
*
* @example <caption>Process all pages.</caption>
// <code>
let count = 0;
for await (const page_data of wiki.allpages({ namespace: 'Talk', apfrom: wiki.remove_namespace('ABC') })) {
if (++count > 5) break;
console.trace('page_data:', page_data);
}
console.log('Done.');
// </code>
*
*/

// wrapper for sync functions
for (const function_name of ('namespace|remove_namespace|is_namespace|to_namespace|is_talk_namespace|to_talk_page|talk_page_to_main|normalize_title|redirect_target_of|aliases_of_page|is_template'
for (const function_name of ('namespace|remove_namespace|is_article|is_namespace|to_namespace|is_talk_namespace|to_talk_page|talk_page_to_main|normalize_title|redirect_target_of|aliases_of_page|is_template'
// CeL.run('application.net.wiki.featured_content');
// [].map(wiki.to_talk_page.bind(wiki))
+ '|get_featured_content_configurations').split('|')) {
Expand All @@ -1992,6 +2020,41 @@ for (const type of wiki_API.list.type_list) {
}
const _this = this;
//console.trace(arguments);

let done, page_queue = [], resolve_queue, waiting_promise, waiting_resolve;
function for_each_page(page_data) {
if (page_queue.abort)
return CeL.wiki.list.exit;

//console.trace(page_data, done, page_queue, resolve_queue);
if (original_for_each_page)
original_for_each_page.apply(this, arguments);

if (!resolve_queue)
return;

if (resolve_queue.length === 0) {
page_queue.push(page_data);
//console.log(page_queue.length, 'pages in queue');
} else {
if (page_queue.length > 0) {
page_queue.push(page_data);
// 由最早的開始給。
page_data = page_queue.shift();
}
resolve_queue.shift()({ value: page_data });
}

if (page_queue.length > 100) {
if (waiting_resolve)
waiting_resolve();
return new Promise(resolve => { waiting_resolve = resolve; });
}
}

const original_for_each_page = options?.for_each_page;
options = { ...options, for_each_page, get_list: options?.get_list || !original_for_each_page };

/**
* @example <code>
Expand All @@ -2002,16 +2065,55 @@ for (const type of wiki_API.list.type_list) {
*/
const promise = Wikiapi_list.call(this, type, title, options)
.then((page_list) => {
// console.log(page_list);
// console.trace(page_list);
//console.trace(page_list.length, 'pages');
//console.trace(page_queue, resolve_queue);
done = true;
page_list.each = Wikiapi_for_each_page.bind(_this, page_list);
return page_list;
});

promise[Symbol.iterator] = () => {
// e.g., `for (const page_data of wiki.allredirects()) { console.log(page_data); }`

// TODO: 將獲得整個list最後才回傳，改成獲得一筆資料回傳一次。不保留已回傳過的資料，以節省記憶體使用。
throw new Error('NYI');
// 依照標準實作，會先執行一次 next()。之後待 for_each_page() 將此 betch 全 push 進 page_queue 後，再依序 call next()。因此 resolve_queue.length <= 1。
// @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols
promise[Symbol.asyncIterator] = () => {
resolve_queue = [];

return {
next() {
//console.trace(done, page_queue, resolve_queue);
if (done && page_queue.length === 0)
return { done };
return new Promise( /* executor */ function (resolve, reject) {
//console.trace(done, page_queue, resolve_queue);
if (page_queue.length === 0) {
resolve_queue.push(resolve);
if (waiting_resolve) {
waiting_resolve();
waiting_resolve = null;
}

} else {
if (resolve_queue.length > 0) {
// 依照標準實作不會到這裡來。
resolve_queue.push(resolve);
// 由最早的開始餵。
resolve = resolve_queue.shift();
}
resolve({ value: page_queue.shift() });
}
});
},
return() {
// e.g., break loop
page_queue.abort = done = true;
return { done };
},
throw(error) {
CeL.error('Wikiapi TODO: yet tested');
page_queue.abort = done = true;
return { done };
}
};
}

return promise;
Expand Down
34 changes: 17 additions & 17 deletions docs/Wikiapi.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ <h4 class="name" id=".KEY_subcategories"><span class="type-signature">(static) <

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1160">line 1160</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1161">line 1161</a>
</li></ul></dd>


Expand Down Expand Up @@ -380,7 +380,7 @@ <h4 class="name" id="category_tree"><span class="type-signature"></span>category

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1136">line 1136</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1137">line 1137</a>
</li></ul></dd>


Expand Down Expand Up @@ -600,7 +600,7 @@ <h4 class="name" id="convert_Chinese"><span class="type-signature"></span>conver

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1669">line 1669</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1670">line 1670</a>
</li></ul></dd>


Expand Down Expand Up @@ -1084,7 +1084,7 @@ <h4 class="name" id="delete"><span class="type-signature"></span>delete<span cla

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1843">line 1843</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1844">line 1844</a>
</li></ul></dd>


Expand Down Expand Up @@ -1296,7 +1296,7 @@ <h4 class="name" id="download"><span class="type-signature"></span>download<span

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1521">line 1521</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1522">line 1522</a>
</li></ul></dd>


Expand Down Expand Up @@ -1551,7 +1551,7 @@ <h4 class="name" id="edit"><span class="type-signature"></span>edit<span class="

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1891">line 1891</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1892">line 1892</a>
</li></ul></dd>


Expand Down Expand Up @@ -2039,7 +2039,7 @@ <h4 class="name" id="for_each_page"><span class="type-signature"></span>for_each

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1594">line 1594</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1595">line 1595</a>
</li></ul></dd>


Expand Down Expand Up @@ -2318,7 +2318,7 @@ <h4 class="name" id="get_featured_content"><span class="type-signature"></span>g

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1762">line 1762</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1763">line 1763</a>
</li></ul></dd>


Expand Down Expand Up @@ -2506,7 +2506,7 @@ <h4 class="name" id="listen"><span class="type-signature"></span>listen<span cla

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1924">line 1924</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1925">line 1925</a>
</li></ul></dd>


Expand Down Expand Up @@ -3738,7 +3738,7 @@ <h5>Examples</h5>
const page_data = await wiki.page('JavaScript');
const parsed = page_data.parse();
let infobox;
// Read Infobox templates, convert to JSON.
// Read [[w:en:MOS:INFOBOX|Infobox templates]], convert to JSON.
parsed.each('template', template_token => {
if (template_token.name.startsWith('Infobox')) {
infobox = template_token.parameters;
Expand Down Expand Up @@ -4328,7 +4328,7 @@ <h4 class="name" id="redirects_here"><span class="type-signature"></span>redirec

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1252">line 1252</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1253">line 1253</a>
</li></ul></dd>


Expand Down Expand Up @@ -4538,7 +4538,7 @@ <h4 class="name" id="redirects_root"><span class="type-signature"></span>redirec

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1215">line 1215</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1216">line 1216</a>
</li></ul></dd>


Expand Down Expand Up @@ -4748,7 +4748,7 @@ <h4 class="name" id="register_redirects"><span class="type-signature"></span>reg

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1335">line 1335</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1336">line 1336</a>
</li></ul></dd>


Expand Down Expand Up @@ -5004,7 +5004,7 @@ <h4 class="name" id="search"><span class="type-signature"></span>search<span cla

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1181">line 1181</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1182">line 1182</a>
</li></ul></dd>


Expand Down Expand Up @@ -5215,7 +5215,7 @@ <h4 class="name" id="site_name"><span class="type-signature"></span>site_name<sp

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1816">line 1816</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1817">line 1817</a>
</li></ul></dd>


Expand Down Expand Up @@ -5899,7 +5899,7 @@ <h4 class="name" id="upload"><span class="type-signature"></span>upload<span cla

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1442">line 1442</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1443">line 1443</a>
</li></ul></dd>


Expand Down Expand Up @@ -6143,7 +6143,7 @@ <h5>Returns:</h5>
<br class="clear">

<footer>
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.10</a> on Fri Dec 22 2023 06:05:01 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.10</a> on Tue Jan 16 2024 06:30:49 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
</footer>

<script>prettyPrint();</script>
Expand Down
6 changes: 3 additions & 3 deletions docs/global.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1388,7 +1388,7 @@ <h4 class="name" id="Wikiapi_for_each_page_in_list"><span class="type-signature"

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1095">line 1095</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1096">line 1096</a>
</li></ul></dd>


Expand Down Expand Up @@ -1658,7 +1658,7 @@ <h4 class="name" id="Wikiapi_list"><span class="type-signature"></span>Wikiapi_l

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1024">line 1024</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1025">line 1025</a>
</li></ul></dd>


Expand Down Expand Up @@ -1770,7 +1770,7 @@ <h5>Examples</h5>
<br class="clear">

<footer>
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.10</a> on Fri Dec 22 2023 06:05:01 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.10</a> on Tue Jan 16 2024 06:30:49 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
</footer>

<script>prettyPrint();</script>
Expand Down
2 changes: 1 addition & 1 deletion docs/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,7 @@ <h2>
<br class="clear">

<footer>
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.10</a> on Fri Dec 22 2023 06:05:01 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.10</a> on Tue Jan 16 2024 06:30:49 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
</footer>

<script>prettyPrint();</script>
Expand Down
Loading

0 comments on commit b65a3cd

Please sign in to comment.