<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<?covid-19-tdm?>
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR Human Factors</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Hum Factors</journal-id>
      <journal-title>JMIR Human Factors</journal-title>
      <issn pub-type="epub">2292-9495</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v10i1e43120</article-id>
      <article-id pub-id-type="pmid">37290040</article-id>
      <article-id pub-id-type="doi">10.2196/43120</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Democratizing the Development of Chatbots to Improve Public Health: Feasibility Study of COVID-19 Misinformation</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Kushniruk</surname>
            <given-names>Andre</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Espinoza</surname>
            <given-names>Juan</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Chang</surname>
            <given-names>Angela</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Taba</surname>
            <given-names>Melody</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author">
          <name name-style="western">
            <surname>Powell</surname>
            <given-names>Leigh</given-names>
          </name>
          <degrees>EdD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-6714-7371</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Nour</surname>
            <given-names>Radwa</given-names>
          </name>
          <degrees>DDS</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-8133-1562</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Sleibi</surname>
            <given-names>Randa</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-0448-1211</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Al Suwaidi</surname>
            <given-names>Hanan</given-names>
          </name>
          <degrees>MBBS</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1261-4224</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Zary</surname>
            <given-names>Nabil</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Institute for Excellence in Health Professions Education</institution>
            <institution>Mohammed Bin Rashid University of Medicine and Health Sciences</institution>
            <addr-line>Building 14, Dubai Healthcare City</addr-line>
            <addr-line>PO Box 505055</addr-line>
            <addr-line>Dubai</addr-line>
            <country>United Arab Emirates</country>
            <phone>971 585960762</phone>
            <email>nabil.zary@mbru.ac.ae</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-8999-6999</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Institute for Excellence in Health Professions Education</institution>
        <institution>Mohammed Bin Rashid University of Medicine and Health Sciences</institution>
        <addr-line>Dubai</addr-line>
        <country>United Arab Emirates</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>College of Medicine</institution>
        <institution>Mohammed Bin Rashid University of Medicine and Health Sciences</institution>
        <addr-line>Dubai</addr-line>
        <country>United Arab Emirates</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Nabil Zary <email>nabil.zary@mbru.ac.ae</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <year>2023</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>28</day>
        <month>12</month>
        <year>2023</year>
      </pub-date>
      <volume>10</volume>
      <elocation-id>e43120</elocation-id>
      <history>
        <date date-type="received">
          <day>30</day>
          <month>9</month>
          <year>2022</year>
        </date>
        <date date-type="rev-request">
          <day>3</day>
          <month>11</month>
          <year>2022</year>
        </date>
        <date date-type="rev-recd">
          <day>5</day>
          <month>1</month>
          <year>2023</year>
        </date>
        <date date-type="accepted">
          <day>7</day>
          <month>6</month>
          <year>2023</year>
        </date>
      </history>
      <copyright-statement>©Leigh Powell, Radwa Nour, Randa Sleibi, Hanan Al Suwaidi, Nabil Zary. Originally published in JMIR Human Factors (https://humanfactors.jmir.org), 28.12.2023.</copyright-statement>
      <copyright-year>2023</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Human Factors, is properly cited. The complete bibliographic information, a link to the original publication on https://humanfactors.jmir.org, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://humanfactors.jmir.org/2023/1/e43120" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Chatbots enable users to have humanlike conversations on various topics and can vary widely in complexity and functionality. An area of research priority in chatbots is democratizing chatbots to all, removing barriers to entry, such as financial ones, to help make chatbots a possibility for the wider global population to improve access to information, help reduce the digital divide between nations, and improve areas of public good (eg, health communication). Chatbots in this space may help create the potential for improved health outcomes, potentially alleviating some of the burdens on health care providers and systems to be the sole voices of outreach to public health.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study explored the feasibility of developing a chatbot using approaches that are accessible in low- and middle-resource settings, such as using technology that is low cost, can be developed by nonprogrammers, and can be deployed over social media platforms to reach the broadest-possible audience without the need for a specialized technical team.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>This study is presented in 2 parts. First, we detailed the design and development of a chatbot, VWise, including the resources used and development considerations for the conversational model. Next, we conducted a case study of 33 participants who engaged in a pilot with our chatbot. We explored the following 3 research questions: (1) Is it feasible to develop and implement a chatbot addressing a public health issue with only minimal resources? (2) What is the participants’ experience with using the chatbot? (3) What kinds of measures of engagement are observed from using the chatbot?</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>A high level of engagement with the chatbot was demonstrated by the large number of participants who stayed with the conversation to its natural end (n=17, 52%), requested to see the free online resource, selected to view all information about a given concern, and returned to have a dialogue about a second concern (n=12, 36%).</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>This study explored the feasibility of and the design and development considerations for a chatbot, VWise. Our early findings from this initial pilot suggest that developing a functioning and low-cost chatbot is feasible, even in low-resource environments. Our results show that low-resource environments can enter the health communication chatbot space using readily available human and technical resources. However, despite these early indicators, many limitations exist in this study and further work with a larger sample size and greater diversity of participants is needed. This study represents early work on a chatbot in its virtual infancy. We hope this study will help provide those who feel chatbot access may be out of reach with a useful guide to enter this space, enabling more democratized access to chatbots for all.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>COVID-19</kwd>
        <kwd>vaccine hesitancy</kwd>
        <kwd>infodemic</kwd>
        <kwd>chatbot</kwd>
        <kwd>motivational interviewing</kwd>
        <kwd>social media</kwd>
        <kwd>conversational agent</kwd>
        <kwd>misinformation</kwd>
        <kwd>online health information</kwd>
        <kwd>usability study</kwd>
        <kwd>vaccine misinformation</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background</title>
        <p>Chatbots are becoming more commonplace in our daily lives, especially in fields such as consumer marketing, customer support, education, and health care, and have significantly increased in recent years [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref2">2</xref>]. Chatbots enable users to have humanlike conversations on various topics. They can vary widely in complexity and functionality, ranging from simple information-giving chatbots to those using artificial intelligence to understand human language input [<xref ref-type="bibr" rid="ref3">3</xref>]. There is a great diversity of chatbots available today, ranging in abilities, features, and complexity. A hierarchy of chatbots has been proposed [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref5">5</xref>], which classifies chatbots according to their ability to use algorithms or artificial intelligence or both to recognize the context of language as it is written in a real-time conversation, called natural language processing (NLP), and respond with greater specificity. Chatbots higher on the classification scale have more advanced NLP, promoting a greater likelihood of mimicking an actual conversation. The more advanced the chatbot, the greater the need for specialized technical expertise to build and maintain it.</p>
        <p>Chatbot development platforms have come a long way, enabling those without technical expertise to use visually interactive approaches to develop chatbots and provide a simplified deployment approach that allows easy integration into social media platforms [<xref ref-type="bibr" rid="ref6">6</xref>]. As technology advances, the opportunity for low- and middle-resource environments to move into the chatbot space increases. In their study describing emerging research needs in chatbots, Følstad et al [<xref ref-type="bibr" rid="ref7">7</xref>] stated that a priority area is democratizing chatbots to all. Democratizing chatbots means removing barriers to entry, such as financial resources, technical resources, or specialized human resources, to help make chatbots a possibility for the wider global population. The goal of democratizing chatbots is to improve access to information, help reduce the digital divide between nations, and improve areas of public good [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref7">7</xref>].</p>
        <p>An area that has seen expanded activity in recent years is chatbots for health care and health communication. Chatbots are increasingly used in health care to address various concerns, from simple to complex. Chatbots in the health space are typically domain specific, deployed for a particular area of focus [<xref ref-type="bibr" rid="ref2">2</xref>]. Common uses for chatbots include simple tasks, such as providing tracking and reminders to support medication and appointment adherence [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref9">9</xref>]. However, more advanced chatbots are implemented to support and promote more complex health concerns, such as mental health support [<xref ref-type="bibr" rid="ref10">10</xref>-<xref ref-type="bibr" rid="ref12">12</xref>], smoking cessation [<xref ref-type="bibr" rid="ref13">13</xref>], and promoting physical health and nutrition [<xref ref-type="bibr" rid="ref9">9</xref>].</p>
        <p>Chatbots that address these more complex health concerns often integrate a behavior change model into the conversation. This ensures that the chatbot does not simply deliver information, the least efficient way to impact health behaviors [<xref ref-type="bibr" rid="ref14">14</xref>], but converses with the participant to maximize the opportunity for behavior change. These chatbots are often more advanced in functionality, such as NLP. The more advanced the chatbot, the greater the need for specialized technical and human resources, creating additional costs, which restrict chatbots to only affluent nations that can fund such projects. However, with the advent of simplified and low-cost development platforms enabling even nonprogrammers to build chatbots, there is an opportunity to democratize chatbots to all nations, including those with low resources. In these nations, health communication chatbots may make the most impact, potentially alleviating some of the burden on health care providers and health care systems to be the sole voices of outreach to public health. For example, studies have reported building a chatbot that asks diagnosis questions to help rule out or detect possible COVID-19 cases, thereby reducing the number of patients coming into primary care [<xref ref-type="bibr" rid="ref15">15</xref>], or chatbots that can diagnose a disease and provide some information about it before consulting a physician, thereby reducing health care costs and providing medical information from a credible source [<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref17">17</xref>]. Other chatbots aim to promote healthy lifestyles and public health education by delivering nutritional education [<xref ref-type="bibr" rid="ref18">18</xref>] and continued care at home for geriatric patients after hospital discharge [<xref ref-type="bibr" rid="ref19">19</xref>]. All these chatbots and more serve as virtual assistants ensuring patient care and education without burdening health care systems.</p>
        <p>Our domain of focus is COVID-19 vaccine misinformation. The public’s ability to receive information, communicate their needs, connect with others, and mobilize community engagement are all factors that can impact the success of health communication initiatives [<xref ref-type="bibr" rid="ref20">20</xref>]. The COVID-19 pandemic is an example of a worldwide impact in which social media was used to propagate misinformation regarding the virus and the vaccine [<xref ref-type="bibr" rid="ref21">21</xref>]. Inaccurate and false information severely impacts public health, delaying individual health choices to take preventative measures, such as mask wearing or social distancing, and having broader impacts on vaccine uptake [<xref ref-type="bibr" rid="ref22">22</xref>]. Even considering multiple personal and business accounts, a significant number of the population use and interact on social media [<xref ref-type="bibr" rid="ref23">23</xref>]. Social media users increasingly use these platforms as information sources, rapidly consuming and sharing information [<xref ref-type="bibr" rid="ref24">24</xref>]. Misinformation about COVID-19 has been prevalent on social media since the start of the pandemic [<xref ref-type="bibr" rid="ref25">25</xref>], negatively impacting public trust in new COVID-19 vaccines and delaying, and even denying, uptake in various communities [<xref ref-type="bibr" rid="ref22">22</xref>].</p>
      </sec>
      <sec>
        <title>Objectives</title>
        <p>This study explores the feasibility of developing a chatbot using approaches that are accessible in low- and middle-resource settings. These approaches include using technology that is low cost, can be developed by nonprogrammers, and can be deployed over social media platforms to reach the broadest-possible audience. In addition, the technology used does not require a specialized technical team, uses freely available and accurate knowledge bases, and is developed using evidence-based practices to create a conversational model that integrates the potential for a change in health behaviors.</p>
        <p>The paper answers the following research questions:</p>
        <list list-type="bullet">
          <list-item>
            <p>Is developing and implementing a chatbot addressing a public health issue with only minimal resources feasible?</p>
          </list-item>
          <list-item>
            <p>What is the participants’ experience with using the chatbot?</p>
          </list-item>
          <list-item>
            <p>What kinds of measures of engagement are observed from using the chatbot?</p>
          </list-item>
        </list>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Ethical Considerations</title>
        <p>The Mohammed Bin Rashid University (MBRU) Institutional Review Board (IRB) approved this study (approval no. MBRU IRB-2021-67). Consent was obtained from the participants.</p>
      </sec>
      <sec>
        <title>Study Design</title>
        <p>Low-resource environments face a myriad of challenges that can prevent them from entering the chatbot space. These include lacking available human, technical, or specialized resources. Another barrier of entry is the lack of exposed details about how to design a chatbot, an area not often elaborated on in published studies. This section reports on the considerations for the technical environment, the project team, the process of conversation design, and implementation on the technology platform.</p>
      </sec>
      <sec>
        <title>Choosing a Platform</title>
        <p>We carefully reviewed several technology platforms using the following requirements: (1) low recurring monetary costs, (2) a simplified development interface that could be easily used by an individual with little to no technical expertise, and (3) cloud hosting to enable easy deployment and avoid the need for specialized hardware. As a result, we selected ManyChat [<xref ref-type="bibr" rid="ref26">26</xref>], a cloud-based platform with an easy-to-use interface, simple and direct integration into social media platforms, and low and predictable recurring costs.</p>
        <p>ManyChat simplifies conversation development using interactive visual displays of conversational decision trees, enabling users to drag actions and responses (see <xref rid="figure1" ref-type="fig">Figure 1</xref>). ManyChat also seamlessly integrates into major social media platforms (ie, Instagram, Telegram, WhatsApp, Facebook Messenger). We selected Facebook Messenger as our deployment platform because it is freely available and widely used worldwide and Google Sheets as a means to collect data from each conversation.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Example of conversational connections built using the ManyChat user interface.</p>
          </caption>
          <graphic xlink:href="humanfactors_v10i1e43120_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Conversation Design Process</title>
        <p>The core project team for this study consisted of 2 educational experts and 1 research assistant, with consultation advice from 2 health professionals with subject matter expertise in vaccinations and 1 researcher with a focus on computer science.</p>
        <sec>
          <title>Behavior Change Model</title>
          <p>The purpose of any health communication initiative is to change behavior; the aim is the same when using the medium of a chatbot. Integrating a behavior change model into conversation design is an emerging trend in chatbots designed to promote health communication [<xref ref-type="bibr" rid="ref27">27</xref>-<xref ref-type="bibr" rid="ref30">30</xref>]. Although behavior change was outside this study’s scope, we wanted to ensure that the foundations of behavior change were integrated into our conversation model. Many models have been established and tested for patient education and behavioral change. Motivational interviewing (MI) is a behavioral change model used by health care professionals and has been found to be effective. MI has also been used as a foundation in chatbots [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref13">13</xref>]. The MI process includes asking questions to elicit participants’ statements about their beliefs. Conversations in MI are examined by looking at the participants’ statements and identifying them as being indicative of “change talk” or “resistance” [<xref ref-type="bibr" rid="ref14">14</xref>,<xref ref-type="bibr" rid="ref31">31</xref>]. MI has a 4-phase approach: engaging, focusing, evoking, and planning [<xref ref-type="bibr" rid="ref14">14</xref>]. Engaging aims at building a rapport with the participant. Focusing allows the determination of the problem or the identification of the concern. Evoking is when change talk is investigated, and planning reinforces commitment and actions.</p>
        </sec>
        <sec>
          <title>Chatbot and Participant Persona Development</title>
          <p>We collectively developed personas for both the chatbot and potential participants [<xref ref-type="bibr" rid="ref32">32</xref>,<xref ref-type="bibr" rid="ref33">33</xref>].</p>
        </sec>
        <sec>
          <title>Chatbot</title>
          <p>Characteristics of the chatbot persona included name, gender, personality, and communication style (<xref rid="figure2" ref-type="fig">Figure 2</xref>). We selected to use a robot persona as a physical representation, owing to the multicultural environment in the United Arab Emirates, in which many different cultures and styles of dress are seen based on nationality or religious affiliation. Therefore, we named our chatbot VWise. Following VWise’s persona, we developed a list of affirmations, demographic details, and jokes to implement during the conversation.</p>
          <fig id="figure2" position="float">
            <label>Figure 2</label>
            <caption>
              <p>Persona for VWise.</p>
            </caption>
            <graphic xlink:href="humanfactors_v10i1e43120_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
        <sec>
          <title>Participants</title>
          <p>We also developed and used participant personas to guide our early dialogues. As personas are meant to be grounded in real data, we determined characteristics collaboratively using real-life examples of people we had encountered. These characteristics included variations in vaccination status, perspectives about vaccination, vaccine knowledge, sources of information, gender, age, and comfort with technology (<xref rid="figure3" ref-type="fig">Figure 3</xref>) [<xref ref-type="bibr" rid="ref33">33</xref>].</p>
          <fig id="figure3" position="float">
            <label>Figure 3</label>
            <caption>
              <p>Examples of user personas guiding our conversation development.</p>
            </caption>
            <graphic xlink:href="humanfactors_v10i1e43120_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
        <sec>
          <title>Sample Dialogues</title>
          <p>Using our participant and chatbot personas, we conducted mock conversations, where one team member was assigned the role of a persona and another member played the role of the chatbot. These early conversations were used as a test bed to help them understand the flow of a conversation and help us develop chatbot utterances, affirmations, and refine VWise’s personality. The remaining team members took notes, and all conversations were recorded and transcribed. Although behavior change was beyond the scope of this study, promoting behavior change was the ultimate goal, and so our conversation model was built with this in mind. MI is rooted in empathy toward patients. Therefore, it was important for us to design a conversation sequence that ensured conversations affirmed what the participants said, and expressed empathy before asking evoking questions or presenting new information.</p>
          <p>Several iterations of sample dialogues were created, and mock conversations were held with volunteer colleagues outside the research team, which were also recorded, transcribed, and coded similarly.</p>
        </sec>
        <sec>
          <title>Conversation Tracks and Personalization</title>
          <p>Based on the variety of COVID-19 vaccination concerns that arose during mock conversations, we decided to limit the conversation to mRNA vaccines. Creating a knowledge base for a chatbot can be a resource-intensive endeavor [<xref ref-type="bibr" rid="ref34">34</xref>,<xref ref-type="bibr" rid="ref35">35</xref>]. As such, we used freely available frequently asked questions (FAQs) from the World Health Organization (WHO) [<xref ref-type="bibr" rid="ref36">36</xref>] and the Centers for Disease Control and Prevention (CDC) [<xref ref-type="bibr" rid="ref37">37</xref>] for our knowledge base. These FAQs were rewritten into a conversational style during the design process, and conversation tracks were developed based on each concern. The rewritten information was then reviewed by 2 health professionals with expertise in vaccinations to ensure accuracy of the information.</p>
          <p>We addressed 5 concerns related to mRNA COVID-19 vaccines through VWise in this pilot:</p>
          <list list-type="bullet">
            <list-item>
              <p>mRNA vaccines were developed and approved too quickly.</p>
            </list-item>
            <list-item>
              <p>Are mRNA vaccines safe?</p>
            </list-item>
            <list-item>
              <p>What is mRNA?</p>
            </list-item>
            <list-item>
              <p>mRNA might change my DNA.</p>
            </list-item>
            <list-item>
              <p>mRNA might cause fertility issues.</p>
            </list-item>
          </list>
          <p>A conversation track was built for each of these concerns, and each was put into a sample dialogue and trialed with volunteer colleagues. Colleagues included native and nonnative English speakers, which enabled us to refine our language, phrasing, and the chatbot’s personality to help make VWise accessible to the broadest-possible audience. Using sample dialogues also aided in refining the flow of the conversation to help make it more personal and engaging to the participant. Personalizing elements of chatbot conversations promote engagement in chatbots used in health education and commercially [<xref ref-type="bibr" rid="ref38">38</xref>,<xref ref-type="bibr" rid="ref39">39</xref>].</p>
          <p>NLP is not yet feasible for those without access to specialized technical resources. So, our conversation was designed to leverage variables and branching to achieve some personalization. Elements that were personalized included vaccination status (ie, conversation tracks based on fully vaccinated, partially vaccinated, unvaccinated status) as well as answers to demographic questions to build rapport (<xref rid="figure4" ref-type="fig">Figures 4</xref>-<xref rid="figure6" ref-type="fig">6</xref>).</p>
          <fig id="figure4" position="float">
            <label>Figure 4</label>
            <caption>
              <p>Conversation flow storing the vaccination status for later conversation tracks.</p>
            </caption>
            <graphic xlink:href="humanfactors_v10i1e43120_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
          <fig id="figure5" position="float">
            <label>Figure 5</label>
            <caption>
              <p>Conversation flow storing participant concerns and launching conversation tracks based on the vaccination status.</p>
            </caption>
            <graphic xlink:href="humanfactors_v10i1e43120_fig5.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
          <fig id="figure6" position="float">
            <label>Figure 6</label>
            <caption>
              <p>Screenshot of conversation personalization in VWise for a participant who is not vaccinated.</p>
            </caption>
            <graphic xlink:href="humanfactors_v10i1e43120_fig6.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
          <p>As our conversation tracks developed, we sought to balance the user and chatbot interactions to ensure a free-flowing exchange during the conversation. As such, we chose to start the conversation with an information-seeking question, a process referred to as a “call to action” [<xref ref-type="bibr" rid="ref40">40</xref>]. Further refining of the dialogues included aspects of the user interface, using buttons and multiple-choice options to help account for the lack of NLP. In addition to phrasing questions in ways that enabled participants to express their concerns, fears, and misconceptions, which is a fundamental aspect of MI [<xref ref-type="bibr" rid="ref41">41</xref>], we also delivered affirmations and responses that could be somewhat universal to anything said by the participants [<xref ref-type="bibr" rid="ref42">42</xref>].</p>
        </sec>
      </sec>
      <sec>
        <title>Conversation Design Implemention</title>
        <p>A conversation implemented in VWise consists of multiple-choice logic trees with preplanned answers and conversational branches that allow for some personalization of the conversation without the need for NLP. <xref rid="figure7" ref-type="fig">Figure 7</xref> represents the different stages of a conversation. The first 2 phases (welcome and personalize) use rapport building to learn demographic details about the participant, including vaccination status, and share details about VWise as a character. The participant then identifies the area of concern to be addressed using buttons to represent each concern. Next, VWise addresses the concern by exchanging information, which includes gathering additional details about the participant, such as their perceptions about vaccination and the methods of how they typically receive information. Once the concern is addressed, VWise offers an opportunity to address another concern (<xref rid="figure8" ref-type="fig">Figure 8</xref>), and the conversation loops back until the participant responds “no.” VWise then asks whether to share further information in the form of a free online course, for which the participant can select “yes” or “no.” Finally, the conversation ends with understanding the participant’s perception of the chatbot and the influence of the conversation itself. Data from the conversation are stored and then written to a Google Sheet at the end of each stage.</p>
        <fig id="figure7" position="float">
          <label>Figure 7</label>
          <caption>
            <p>Our conversation design implemented in VWise. CUQ: Chatbot Usability Questionnaire; MI: motivational interviewing.</p>
          </caption>
          <graphic xlink:href="humanfactors_v10i1e43120_fig7.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <fig id="figure8" position="float">
          <label>Figure 8</label>
          <caption>
            <p>Conversation flow in VWise. CUQ: Chatbot Usability Questionnaire.</p>
          </caption>
          <graphic xlink:href="humanfactors_v10i1e43120_fig8.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p>Participant perception was investigated using the Chatbot Usability Questionnaire (CUQ) [<xref ref-type="bibr" rid="ref43">43</xref>], a validated tool that assesses different aspects of usability: chatbot’s personality, onboarding, user experience, and error handling. The questionnaire consists of 16 questions, 8 (50%) related to the positive aspects and 8 (50%) to the negative aspects of chatbot usability. The CUQ was embedded in the chat, and scores were calculated using a Microsoft Excel spreadsheet provided by the questionnaire developers. The CUQ score is calculated out of 100. The CUQ developers have designed it such that the scores are comparable to the System Usability Scale (SUS) [<xref ref-type="bibr" rid="ref44">44</xref>], where scores &#62;68 are considered above average. The process data collected are further described in the Case Study section.</p>
        <p>Next, we present a case study detailing the pilot for VWise.</p>
      </sec>
      <sec>
        <title>Case Study</title>
        <sec>
          <title>Study Design</title>
          <p>This study used a convenience sampling approach in which invitations were sent to colleagues at the institution and to those in the research team’s networks. The final number of participants was 33. Participants were provided with an explanation about the aim of the study, the nature of participation, and a description of how to use the chatbot. A consent form was embedded in the conversation, and only participants who selected “yes” were allowed to proceed with the chat. Those who selected “no” were provided with a link to a free educational resource. Inclusion criteria were adults with English language proficiency, digital literacy, and the ability to provide consent.</p>
        </sec>
        <sec>
          <title>Data Collection and Analysis</title>
          <p>Since ManyChat does not store conversations in their entirety, relevant participant responses were first stored in variables and then mapped to a preconfigured Google Sheet that became our data set for analysis. No identifying data, Facebook profile data, or any background data generated by ManyChat were included in the data set. Two independent researchers deductively coded participant responses to qualitative questions. All other data were quantitatively analyzed.</p>
        </sec>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <p>Our results are reported in 2 sections, (1) user engagement and (2) user experience.</p>
      <sec>
        <title>User Engagement</title>
        <p>This section describes how participants engaged with VWise during the pilot, including their journey through the conversation, concerns selected, and points of attrition.</p>
        <sec>
          <title>Participant Journey</title>
          <p>Here is a narrative representation of <xref rid="figure9" ref-type="fig">Figure 9</xref>, which represents the journey of our pilot participants through VWise, including the areas of attrition at various points in the conversation.</p>
          <fig id="figure9" position="float">
            <label>Figure 9</label>
            <caption>
              <p>Participants’ journey through VWise in the pilot, with attrition rates. CUQ: Chatbot Usability Questionnaire.</p>
            </caption>
            <graphic xlink:href="humanfactors_v10i1e43120_fig9.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
          <p>In total, 33 participants began a conversation with VWise, but 2 (6%) exited the conversation prior to the “concern selection” phase, making our sample size 31 (94%). Of these 31 participants, 10 (32%) exited the conversation during the initial “addressing concern” phase. VWise is designed to address 1 concern at a time, allowing participants to loop back and select another concern. The remaining participants (n=21, 68%) were asked whether they would be interested in receiving a free information source about misinformation. Of these 21 participants, 12 (57%) decided to address another concern. No participants elected to address a third concern.</p>
          <p>Furthermore, 8 (38%) of the 21 participants exited the conversation before the end. All participants chose to address only 1 concern (4, 50%, exited after addressing the concern and 4 (50%) without completing the CUQ). In addition, all 12 participants who selected a second concern completed the conversation up to the “influence” phase, with 9 (75%) reaching the end of the conversation by completing the CUQ.</p>
          <p>In the end, 17 (55%) of 31 participants reached the “influence” phase, with 13 (76%) completing the entire conversation by filling out the CUQ.</p>
        </sec>
        <sec>
          <title>Vaccination Status of Participants</title>
          <p>Overall, we had a higher number of fully vaccinated participants. Of the 31 participants, 18 (58%) were fully vaccinated, 9 (29%) were partially vaccinated, and 4 (13%) were unvaccinated.</p>
          <p>Of the 17 (55%) participants who reached the “influence” phase of the conversation, 11 (65%) were fully vaccinated, 5 (29%) were partially vaccinated, and 1 (6%) was unvaccinated.</p>
        </sec>
        <sec>
          <title>Concerns</title>
          <p>Participants were provided with a list of 5 concerns to choose from in the form of clickable buttons. <xref ref-type="table" rid="table1">Table 1</xref> presents the distribution of concerns among the participants.</p>
          <table-wrap position="float" id="table1">
            <label>Table 1</label>
            <caption>
              <p>Distribution of 5 concerns among the participants (N=33).</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="580"/>
              <col width="420"/>
              <thead>
                <tr valign="top">
                  <td>Concern</td>
                  <td>Participants, n (%)</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>mRNA vaccines were developed and approved too quickly.</td>
                  <td>5 (15)</td>
                </tr>
                <tr valign="top">
                  <td>Are mRNA vaccines safe?</td>
                  <td>3 (9)</td>
                </tr>
                <tr valign="top">
                  <td>What is mRNA?</td>
                  <td>14 (42)</td>
                </tr>
                <tr valign="top">
                  <td>mRNA might change my DNA.</td>
                  <td>5 (15)</td>
                </tr>
                <tr valign="top">
                  <td>mRNA might cause fertility issues.</td>
                  <td>4 (12)</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
          <p>Some of the concerns VWise addressed were quite complex, containing many pieces of relevant information. For these concerns, we divided the large and complex responses in the WHO and CDC data into smaller, more engaging chunks of information. VWise asked the participants whether they would like to hear more information between each chunk of information. We interpreted each affirmative response to provide more information as a measure of engagement. In our analysis, all the participants who selected the concerns “mRNA vaccines were developed and approved too quickly” and “mRNA might cause infertility” opted to receive all 5 chunks of information about the concern.</p>
        </sec>
        <sec>
          <title>Qualitative Indicators of MI</title>
          <sec>
            <title>Change Talk vs Resistance</title>
            <p>VWise asked open-ended questions to hear participants’ perceptions and concerns in their own words, a technique used in MI known as using evoking questions [<xref ref-type="bibr" rid="ref45">45</xref>]. Evoking questions in MI are used to understand a participant’s willingness to change behavior. Although behavior change was outside the scope of this study, we wanted to understand whether our conversation elicited any evidence of “change talk” or “resistance.” Indications of resistance in MI include arguing, interrupting, negating, or ignoring [<xref ref-type="bibr" rid="ref14">14</xref>]. Responses were manually and deductively coded qualitatively by 2 independent researchers, who then discussed their codes to reach consensus.</p>
            <p>The conversation flow provided partially vaccinated and unvaccinated participants with 2 opportunities to elicit a willingness to change:</p>
            <list list-type="bullet">
              <list-item>
                <p>“Has not being vaccinated/not getting your booster shot impacted your life in any way?” Partially vaccinated (9/12, 75%) and unvaccinated (3/12, 25%) participants were asked about the impact of not being vaccinated/fully vaccinated on their life. Options were “yes,” “no,” and “not sure.” VWise asks users who respond with “yes” a follow-up, open-ended question: “In what ways has your life been impacted?” However, all 12 (100%) participants responded with a resistance answer (ie, “no,” “not sure”), so no participant was asked the qualitative follow-up question about how their life was impacted.</p>
              </list-item>
              <list-item>
                <p>“If you were to close your eyes and think about your daily life and routine, in what ways might your life be different if you were to get fully vaccinated?” All 12 (100%) partially vaccinated and unvaccinated participants were asked this question, and responses were coded as either “change talk” or “resistance”. Only 2 (17%) participants expressed any form of change talk, and both were unvaccinated (<xref ref-type="table" rid="table2">Table 2</xref>).</p>
              </list-item>
            </list>
            <table-wrap position="float" id="table2">
              <label>Table 2</label>
              <caption>
                <p>Responses of partially vaccinated and unvaccinated participants responses.</p>
              </caption>
              <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
                <col width="230"/>
                <col width="640"/>
                <col width="130"/>
                <thead>
                  <tr valign="top">
                    <td>Vaccination status</td>
                    <td>Response</td>
                    <td>Code</td>
                  </tr>
                </thead>
                <tbody>
                  <tr valign="top">
                    <td>N<sup>a</sup></td>
                    <td>“I could see my friends and play hockey”</td>
                    <td>C<sup>b</sup></td>
                  </tr>
                  <tr valign="top">
                    <td>N</td>
                    <td>“Cannot think of a way”</td>
                    <td>R<sup>c</sup></td>
                  </tr>
                  <tr valign="top">
                    <td>N</td>
                    <td>“Less PCRs<sup>d</sup>”</td>
                    <td>C</td>
                  </tr>
                  <tr valign="top">
                    <td>P<sup>e</sup></td>
                    <td>“Not sure”</td>
                    <td>R</td>
                  </tr>
                  <tr valign="top">
                    <td>P</td>
                    <td>“Forgetting things more?”</td>
                    <td>R</td>
                  </tr>
                  <tr valign="top">
                    <td>P</td>
                    <td>“No”</td>
                    <td>R</td>
                  </tr>
                  <tr valign="top">
                    <td>P</td>
                    <td>“Don’t know”</td>
                    <td>R</td>
                  </tr>
                  <tr valign="top">
                    <td>P</td>
                    <td>“No”</td>
                    <td>R</td>
                  </tr>
                  <tr valign="top">
                    <td>P</td>
                    <td>“My [life] will be normal as already take[n] 2 doses, that’s why I don’t think that the booster [will] make any difference”</td>
                    <td>R</td>
                  </tr>
                  <tr valign="top">
                    <td>P</td>
                    <td>“I cannot find any difference if still I will be in [the] ICU<sup>f</sup> even with 2 doses”</td>
                    <td>R</td>
                  </tr>
                  <tr valign="top">
                    <td>P</td>
                    <td>“What do you think”</td>
                    <td>R</td>
                  </tr>
                  <tr valign="top">
                    <td>P</td>
                    <td>“Lol”</td>
                    <td>R</td>
                  </tr>
                </tbody>
              </table>
              <table-wrap-foot>
                <fn id="table2fn1">
                  <p><sup>a</sup>N: not vaccinated.</p>
                </fn>
                <fn id="table2fn2">
                  <p><sup>b</sup>C: change talk.</p>
                </fn>
                <fn id="table2fn3">
                  <p><sup>c</sup>R: resistance.</p>
                </fn>
                <fn id="table2fn4">
                  <p><sup>d</sup>PCR: polymerase chain reaction.</p>
                </fn>
                <fn id="table2fn5">
                  <p><sup>e</sup>P: partially vaccinated.</p>
                </fn>
                <fn id="table2fn6">
                  <p><sup>f</sup>ICU: intensive care unit.</p>
                </fn>
              </table-wrap-foot>
            </table-wrap>
          </sec>
          <sec>
            <title>Influence of Conversations</title>
            <p>The planning phase of MI typically involves using the participants’ language to turn their words into action. Given the low-tech nature of the chatbot (ie, no NLP), we asked an open-ended question to try and understand whether any participant might have changed their perception of their concern or getting an mRNA vaccine: “Before you go, I would really like to know how this conversation has influenced your opinion about mRNA. What would you like to share with me? Responses were coded and categorized as positive, negative, or neutral (<xref ref-type="table" rid="table3">Table 3</xref>). Only 17 (52%) participants made it to this stage in the conversation, with 10 (59%; n=6, 60%, fully vaccinated and n=4, 40%, partially vaccinated) expressing a positive opinion, 1 (4%; partially vaccinated) expressing a neutral opinion, and 6 (37%; n=5, 83%, fully vaccinated and n=1, 17%, unvaccinated) not answering.</p>
            <table-wrap position="float" id="table3">
              <label>Table 3</label>
              <caption>
                <p>Coded responses to the question “Before you go, I would really like to know how this conversation has influenced your opinion about mRNA. What would you like to share with me?” (N=17).</p>
              </caption>
              <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
                <col width="200"/>
                <col width="250"/>
                <col width="240"/>
                <col width="310"/>
                <thead>
                  <tr valign="top">
                    <td>Influence of conversation</td>
                    <td>Fully vaccinated (n=11), n (%)</td>
                    <td>Not vaccinated (n=1), n (%)</td>
                    <td>Partially vaccinated (n=5), n (%)</td>
                  </tr>
                </thead>
                <tbody>
                  <tr valign="top">
                    <td>No response</td>
                    <td>5 (45)</td>
                    <td>1 (100)</td>
                    <td>0</td>
                  </tr>
                  <tr valign="top">
                    <td>Neutral</td>
                    <td>0</td>
                    <td>0</td>
                    <td>1 (20)</td>
                  </tr>
                  <tr valign="top">
                    <td>Positive</td>
                    <td>6 (55)</td>
                    <td>0</td>
                    <td>4 (80)</td>
                  </tr>
                  <tr valign="top">
                    <td>Negative</td>
                    <td>0</td>
                    <td>0</td>
                    <td>0</td>
                  </tr>
                </tbody>
              </table>
            </table-wrap>
          </sec>
        </sec>
        <sec>
          <title>User Experience</title>
          <sec>
            <title>Participant Demographics</title>
            <p>Participants were asked their name, age, and location, with names being stored and used to personalize welcome responses. The mean age self-reported by participants (n=28, 85%; n=5, 15%, participants did not respond or provided unreal answers, eg, 99 years) was 36.6 years (SD 10.02). <xref ref-type="table" rid="table4">Table 4</xref> presents the self-reported location details of participants.</p>
            <table-wrap position="float" id="table4">
              <label>Table 4</label>
              <caption>
                <p>Self-reported location of participants (N=33).</p>
              </caption>
              <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
                <col width="500"/>
                <col width="500"/>
                <thead>
                  <tr valign="top">
                    <td>Location</td>
                    <td>Participants, n (%)</td>
                  </tr>
                </thead>
                <tbody>
                  <tr valign="top">
                    <td>Afghanistan</td>
                    <td>1 (3)</td>
                  </tr>
                  <tr valign="top">
                    <td>Bermudas</td>
                    <td>1 (3)</td>
                  </tr>
                  <tr valign="top">
                    <td>Canada</td>
                    <td>2 (6)</td>
                  </tr>
                  <tr valign="top">
                    <td>India</td>
                    <td>1 (3)</td>
                  </tr>
                  <tr valign="top">
                    <td>Morocco</td>
                    <td>4 (12)</td>
                  </tr>
                  <tr valign="top">
                    <td>Nepal</td>
                    <td>1 (3)</td>
                  </tr>
                  <tr valign="top">
                    <td>Switzerland</td>
                    <td>1 (3)</td>
                  </tr>
                  <tr valign="top">
                    <td>Tunis</td>
                    <td>1 (3)</td>
                  </tr>
                  <tr valign="top">
                    <td>United Arab Emirates</td>
                    <td>19 (58)</td>
                  </tr>
                  <tr valign="top">
                    <td>No response</td>
                    <td>2 (6)</td>
                  </tr>
                </tbody>
              </table>
            </table-wrap>
            <p>Fully vaccinated participants accounted for around 58% (n=19) of the sample, 12% (n=4) were not vaccinated, and 30% (n=10) were partially vaccinated. Vaccinated (fully and partially) participants (n=29, 88%) were asked to rate the importance of getting a vaccine on a scale of 0-10. The mean score was 7.5 (SD 2.8), with 72% (n=21) of the 29 participants rating the importance to vaccinate as ≥6 (<xref ref-type="table" rid="table5">Table 5</xref>). Fully vaccinated participants (n=19, 58%) rated the importance of getting a vaccine as ≤5 (n=4, 21%) and ≥6 (n=15, 79%). Partially vaccinated participants (n=10, 30%) rated the importance of getting a vaccine as ≤5 (n=4, 40%) and ≥6 (n=6, 60%). Unvaccinated participants were not asked this question.</p>
            <table-wrap position="float" id="table5">
              <label>Table 5</label>
              <caption>
                <p>Distribution of participant responses to the question “On a scale of 0-10, how important to you was it to get vaccinated (0=not important at all, 10=very important)?” (N=29).</p>
              </caption>
              <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
                <col width="340"/>
                <col width="330"/>
                <col width="330"/>
                <thead>
                  <tr valign="top">
                    <td>Score for perceived importance of getting vaccinated (scale 0-10)</td>
                    <td>Fully vaccinated (n=19), n (%)</td>
                    <td>Partially vaccinated<break/>(n=10), n (%)</td>
                  </tr>
                </thead>
                <tbody>
                  <tr valign="top">
                    <td>≤5</td>
                    <td>4 (21)</td>
                    <td>4 (40)</td>
                  </tr>
                  <tr valign="top">
                    <td>≥6</td>
                    <td>15 (79)</td>
                    <td>6 (60)</td>
                  </tr>
                </tbody>
              </table>
            </table-wrap>
          </sec>
        </sec>
        <sec>
          <title>The Chatbot Usability Questionnaire</title>
          <p>The CUQ was included at the end of the conversation. Of the 17 (52%) participants who concluded the chat, 13 (76%) filled out the CUQ. The mean score was 70.9 (SD 19.4), and the median score was 78.1, with the lowest and highest scores being 34.4 and 95.3, respectively. The mean score was higher than the standard mean SUS score of 68. Five positive aspects of the chatbot’s personality scored ≥4 on a 5-point Likert scale: (1) realistic and engaging personality (mean score 4.1, SD 1.2), (2) welcoming during initial setup (mean score 4.4, SD 0.8), (3) explained its scope and purpose well (mean score 4.3, SD 0.6), (4) was easy to navigate (mean score 4.0, SD 1.2), (5) and was easy to use (mean score 4.3, SD 0.9). The remaining 3 positive aspects scored as follows: (1) understood me well (mean score 3.4, SD 1.3); (2) responses were useful, appropriate, and informative (mean score 3.8, SD 1.1); and (3) coped well with any errors or mistakes (mean score 3.0, SD 1.1). All negative aspects of the chatbot scored &#60;3 (<xref ref-type="table" rid="table6">Table 6</xref>).</p>
          <table-wrap position="float" id="table6">
            <label>Table 6</label>
            <caption>
              <p>Mean CUQ<sup>a</sup> score of each aspect.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="600"/>
              <col width="400"/>
              <thead>
                <tr valign="top">
                  <td>Question</td>
                  <td>Score, mean(SD)</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>The chatbot’s personality was realistic and engaging.</td>
                  <td>4.1 (1.2)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot seemed too robotic.</td>
                  <td>2.8 (1.5)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot was welcoming during initial setup.</td>
                  <td>4.4 (0.8)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot seemed unfriendly.</td>
                  <td>1.5 (0.8)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot explained its scope and purpose well.</td>
                  <td>4.3 (0.6)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot gave no indication as to its purpose.</td>
                  <td>2.1 (1.1)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot was easy to navigate.</td>
                  <td>4.0 (1.2)</td>
                </tr>
                <tr valign="top">
                  <td>It would be easy to get confused when using the chatbot.</td>
                  <td>2.1 (1.1)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot understood me well.</td>
                  <td>3.4 (1.3)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot failed to recognize a lot of my inputs.</td>
                  <td>2.9 (1.5)</td>
                </tr>
                <tr valign="top">
                  <td>Chatbot responses were useful, appropriate, and informative.</td>
                  <td>3.8 (1.1)</td>
                </tr>
                <tr valign="top">
                  <td>Chatbot responses were irrelevant.</td>
                  <td>2.4 (1.2)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot coped well with any errors or mistakes.</td>
                  <td>3.0 (1.1)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot seemed unable to handle any errors.</td>
                  <td>2.5 (1.2)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot was easy to use.</td>
                  <td>4.3 (0.9)</td>
                </tr>
                <tr valign="top">
                  <td>The chatbot was complex.</td>
                  <td>1.6 (0.8)</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table6fn1">
                <p><sup>a</sup>CUQ: Chatbot Usability Questionnaire.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
          <p>In addition to the CUQ, some participants voluntarily sent us their feedback via email. We noted a diversity in the feedback regarding the personality of the chatbot.</p>
          <p>Some of the positive feedback included:</p>
          <disp-quote>
            <p>Bot is friendly</p>
          </disp-quote>
          <disp-quote>
            <p>VWise is quick to respond.</p>
          </disp-quote>
          <disp-quote>
            <p>Engages with the participant</p>
          </disp-quote>
          <disp-quote>
            <p>I like the conversational use of language.</p>
          </disp-quote>
          <disp-quote>
            <p>Clever chatbot with precise answers!</p>
          </disp-quote>
          <p>Not all comments about VWise were positive:</p>
          <disp-quote>
            <p>A bit too friendly</p>
          </disp-quote>
          <disp-quote>
            <p>VWise seems a funny character. Please revisit. [H]ere are areas you can avoid having some funny comments and emojis. Not everyone like too much fun when discussing serious/important information.</p>
          </disp-quote>
          <disp-quote>
            <p>I felt like the intro was too long and a little “extra friendly”. It could be shortened, but the extra friendly could be great if you’re targeting kids and younger adults when sharing information about the vaccine.</p>
          </disp-quote>
          <p>We also received feedback about the content of the conversation from participants. Early positive feedback included:</p>
          <disp-quote>
            <p>It’s good to have informed by this topic so now I have [an] idea of what mean mRNA.</p>
          </disp-quote>
          <disp-quote>
            <p>More confident to take the vaccine</p>
          </disp-quote>
          <p>Criticisms included a desire to be pointed to additional resources:</p>
          <disp-quote>
            <p>When sharing the answer about the relation between taking the vaccine and infertility, it might’ve been better to share a source perhaps [than] just saying “studies show no connection between them”.</p>
          </disp-quote>
          <p>In addition, participants highlighted the need to create a conversation flow that focuses more on a 2-way exchange of interactions:</p>
          <disp-quote>
            <p>When I used it, it felt more the bot wants to know about myself as opposite to I want to use the bot to know more about something.</p>
          </disp-quote>
        </sec>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>This study sought to address the possibility of democratizing chatbot access to all by exploring the feasibility of developing a chatbot for public health communication using readily available resources and technology that would be accessible in low- and middle-resource settings. We explored this through a detailed description of the design and development considerations for our chatbot and by presenting a case study describing our initial pilot with 33 people who engaged in a conversation about COVID-19 vaccine misinformation.</p>
        <p>Our study highlights that even in low-resource environments, the ability to develop a functioning and low-cost chatbot is feasible. A high level of engagement with the chatbot was demonstrated by the large number of participants who stayed with the conversation to its natural end and requested to see the free online resource, who selected to view all information about a given concern, and who returned to have a dialogue about a second concern. More than half of the participants (52%) continued the conversation till the end, and around 36% went for a second chat. These numbers are promising and suggest that with further improvement, the retention rate could be high.</p>
        <p>Participants’ responses to the CUQ were positive. Emailed comments from participants revealed a need to work on the conversational flow as well as the chatbot’s personality (ie, extra friendly, a funny character). The discrepancy in the reception to the chatbot’s personality could be attributed to the cultural diversity of the participants. Nißen et al [<xref ref-type="bibr" rid="ref46">46</xref>] recently noted that demographic differences, especially age, may be a determiner of receptivity to a chatbot, helping determine whether a bond between participant and chatbot might be formed. An emerging area of exploration is the development of personality-adaptive chatbots, in which the chatbot’s personality is tailored to an individual user based on key characteristics [<xref ref-type="bibr" rid="ref47">47</xref>]. Future work on VWise could investigate whether the ability to create personality-adaptive chatbots is conducive to environments in which specialized resources are scarce.</p>
        <p>Although behavior change was outside the scope of this study, we observed participants’ responses that could be construed as willingness or resistance to change. These observations could not be processed in real time (ie, only through post hoc manual coding), so we believe that the results, at the minimum, demonstrate that our conversation flow is promising and stimulates thought processes. This could be a precursor to behavior change. Future iterations should seek to take advantage of the advanced features of ManyChat, such as keyword detection, designed to help simulate an NLP experience. Another avenue for exploration is to use a hybrid approach, in which non-NLP chatbots serve to conduct an initial consultant, helping identify candidates for further intervention. For example, Lee et al [<xref ref-type="bibr" rid="ref48">48</xref>] followed a hybrid approach in which a low-tech chatbot was used as a mediator for patients to self-disclose mental health needs before approaching a mental health professional. In our case study, a wealth of information about participants was collected, making branching scenarios possible, in which specific participants are pushed toward professionals or educational interventions.</p>
        <p>Finally, to help further the aim of democratizing chatbot access for all, we recommend that future studies expose and labor the design and development processes and technology choices for their chatbots to enable others to reproduce their work. When consulting the literature to guide this project, only a few studies elaborated on the process of designing a conversational model or included recommendations for a smooth user experience [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref49">49</xref>,<xref ref-type="bibr" rid="ref50">50</xref>]. Industry is further ahead in this area and was used more as a guide for this study [<xref ref-type="bibr" rid="ref33">33</xref>,<xref ref-type="bibr" rid="ref51">51</xref>-<xref ref-type="bibr" rid="ref53">53</xref>]. Research should seek to catch up to industry by sharing the best practices and processes through published, peer-reviewed work.</p>
      </sec>
      <sec>
        <title>Limitations</title>
        <p>This early study explored the feasibility of an approach to developing chatbots in low-resource environments. However, this study has many limitations.</p>
        <p>This pilot was not a controlled study. Convenience sampling was used, and our sample size was quite small, consisting primarily of fully vaccinated people and with fewer respondents to the CUQ. Therefore, due to the limited scope of this paper (ie, feasibility), our results are largely descriptive, with the CUQ results serving only to aid the research team in areas for improvement. Future iterations should include a larger and more diverse sample to help us obtain a better understanding of the effectiveness and of the improvements needed to the content and conversation design.</p>
        <p>Concerning the subject matter and our approach to branched conversations based on vaccination status, our approach for fully vaccinated participants centered on information delivery. A by-product of addressing vaccine misinformation is to increase vaccination uptake, so fully vaccinated participants do not fall into this category. Chatbots for health communication deployed over social media cannot know ahead of time the characteristics of those who will use them. As such, it is important to undertake persona exercises to understand who your participants might be and include an intended outcome for each. Future studies should use an engagement strategy in which the goal is to empower fully vaccinated participants to share their experiences and information with others via social media, as well as exploring different social media strategies that might attract a greater diversity of individuals to engage with the chatbot.</p>
        <p>Since behavior change was not in the scope of this study, integrating MI as a behavior change model needs to be explored in future iterations. MI was largely selected due to it being a well-published model in the literature about chatbots [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref54">54</xref>,<xref ref-type="bibr" rid="ref55">55</xref>], but there are also those who use an eclectic approach [<xref ref-type="bibr" rid="ref27">27</xref>] or develop and implement their own models [<xref ref-type="bibr" rid="ref28">28</xref>-<xref ref-type="bibr" rid="ref30">30</xref>]. Future work should include the exploration of other models of behavior change based on a larger and more diverse participant population. Additionally, a behavior change expert was not consulted in this study, and future studies would benefit from having this skill set on the development team.</p>
        <p>We were also limited by not being able to conclude that any behavior changed; only indications of change and resistance could be detected, albeit post hoc. Previous research, such as Altay et al [<xref ref-type="bibr" rid="ref56">56</xref>], has shown that behavior change can be detected using a chatbot; however, as the feasibility of democratizing development was the aim of this paper, future work is needed on how this might be achieved using low-cost technology solutions.</p>
      </sec>
      <sec>
        <title>Conclusion</title>
        <p>This study explored the feasibility of and design and development considerations for VWise, a chatbot created to enable a greater diversity of environments to enter the chatbot space by using readily available human and technical resources. Although our findings are descriptive in nature, our pilot of VWise shows promise with regard to whether low-resource environments can enter the health communication chatbot space. The early conversational model also shows promise as many participants followed the conversation to its natural end and many extended the conversation through selection of a second concern. However, improvements to the chatbot’s personality and conversation flow are needed and further pilots with a larger sample size and diversity of vaccination status are necessary. This study represents early work of a chatbot in its virtual infancy. We hope this study will help provide those who feel chatbot access may be out of reach with a useful guide to entering this space, enabling more democratized access to chatbots for all.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">CDC</term>
          <def>
            <p>Centers for Disease Control and Prevention</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">CUQ</term>
          <def>
            <p>Chatbot Usability Questionnaire</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">FAQ</term>
          <def>
            <p>frequently asked question</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">MI</term>
          <def>
            <p>motivational interviewing</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">NLP</term>
          <def>
            <p>natural language processing</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">SUS</term>
          <def>
            <p>System Usability Scale</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb7">WHO</term>
          <def>
            <p>World Health Organization</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This work was partly supported by the Al Jalila Foundation and, in addition, funded by a Pfizer Independent Medical Education Grant (agreement 67504787).</p>
      <p>We wish to thank Dr Youness Zidoun and Sreelekshmi Kaladhara for their significant contributions to the vaccine hesitancy project.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>NZ was Editor-in-Chief of JMIR Serious Games and JMIR Medical Education at the time of submission and acceptance of this publication.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Adamopoulou</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Moussiades</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Maglogiannis</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Iliadis</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Pimenidis</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>An overview of chatbot technology</article-title>
          <source>Artificial Intelligence Applications and Innovations</source>
          <year>2020</year>
          <publisher-loc>Cham</publisher-loc>
          <publisher-name>Springer International Publishing</publisher-name>
          <fpage>373</fpage>
          <lpage>383</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jovanovic</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Baez</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Casati</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>Chatbots as conversational healthcare services</article-title>
          <source>IEEE Internet Comput</source>
          <year>2021</year>
          <month>5</month>
          <volume>25</volume>
          <issue>3</issue>
          <fpage>44</fpage>
          <lpage>51</lpage>
          <pub-id pub-id-type="doi">10.1109/mic.2020.3037151</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ramesh</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Ravishankaran</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Joshi</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Chandrasekaran</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Kaushik</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Gupta</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Kharb</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Chahal</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>A survey of design techniques for conversational agents</article-title>
          <source>Information, Communication and Computing Technology</source>
          <year>2017</year>
          <publisher-loc>Singapore</publisher-loc>
          <publisher-name>Springer</publisher-name>
          <fpage>336</fpage>
          <lpage>350</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Powell</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Nizam</surname>
              <given-names>MZ</given-names>
            </name>
            <name name-style="western">
              <surname>Nour</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Zidoun</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Sleibi</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Kaladhara Warrier</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Al Suwaidi</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Zary</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Conversational agents in health education: protocol for a scoping review</article-title>
          <source>JMIR Res Protoc</source>
          <year>2022</year>
          <month>04</month>
          <day>19</day>
          <volume>11</volume>
          <issue>4</issue>
          <fpage>e31923</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.researchprotocols.org/2022/4/e31923/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/31923</pub-id>
          <pub-id pub-id-type="medline">35258006</pub-id>
          <pub-id pub-id-type="pii">v11i4e31923</pub-id>
          <pub-id pub-id-type="pmcid">PMC9066353</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="web">
          <article-title>5 Levels of conversational AI - 2020 update</article-title>
          <source>Rasa</source>
          <access-date>2022-08-30</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://rasa.com blog/5-levels-of-conversational-ai-2020-update/">https://rasa.com blog/5-levels-of-conversational-ai-2020-update/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rodriguez-Gil</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Garcia-Zubia</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Orduna</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Villar-Martinez</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Lopez-De-Ipina</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>New approach for conversational agent definition by non-programmers: a visual domain-specific language</article-title>
          <source>IEEE Access</source>
          <year>2019</year>
          <volume>7</volume>
          <fpage>5262</fpage>
          <lpage>5276</lpage>
          <pub-id pub-id-type="doi">10.1109/access.2018.2883500</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Følstad</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Araujo</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Law</surname>
              <given-names>EL</given-names>
            </name>
            <name name-style="western">
              <surname>Brandtzaeg</surname>
              <given-names>PB</given-names>
            </name>
            <name name-style="western">
              <surname>Papadopoulos</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Reis</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Baez</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Laban</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>McAllister</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Ischen</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Wald</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Catania</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Meyer von Wolff</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Hobert</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Luger</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Future directions for chatbot research: an interdisciplinary research agenda</article-title>
          <source>Computing</source>
          <year>2021</year>
          <month>10</month>
          <day>19</day>
          <volume>103</volume>
          <issue>12</issue>
          <fpage>2915</fpage>
          <lpage>2942</lpage>
          <pub-id pub-id-type="doi">10.1007/s00607-021-01016-7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Echeazarra</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Pereira</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Saracho</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>TensioBot: a chatbot assistant for self-managed in-house blood pressure checking</article-title>
          <source>J Med Syst</source>
          <year>2021</year>
          <month>03</month>
          <day>15</day>
          <volume>45</volume>
          <issue>4</issue>
          <fpage>54</fpage>
          <pub-id pub-id-type="doi">10.1007/s10916-021-01730-x</pub-id>
          <pub-id pub-id-type="medline">33723721</pub-id>
          <pub-id pub-id-type="pii">10.1007/s10916-021-01730-x</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bickmore</surname>
              <given-names>TW</given-names>
            </name>
            <name name-style="western">
              <surname>Schulman</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Sidner</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Automated interventions for multiple health behaviors using conversational agents</article-title>
          <source>Patient Educ Couns</source>
          <year>2013</year>
          <month>08</month>
          <volume>92</volume>
          <issue>2</issue>
          <fpage>142</fpage>
          <lpage>148</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/23763983"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.pec.2013.05.011</pub-id>
          <pub-id pub-id-type="medline">23763983</pub-id>
          <pub-id pub-id-type="pii">S0738-3991(13)00211-5</pub-id>
          <pub-id pub-id-type="pmcid">PMC3727973</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Park</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Oh</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>La</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Suh</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>Designing a chatbot for a brief motivational interview on stress management: qualitative case study</article-title>
          <source>J Med Internet Res</source>
          <year>2019</year>
          <month>04</month>
          <day>16</day>
          <volume>21</volume>
          <issue>4</issue>
          <fpage>e12231</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2019/4/e12231/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/12231</pub-id>
          <pub-id pub-id-type="medline">30990463</pub-id>
          <pub-id pub-id-type="pii">v21i4e12231</pub-id>
          <pub-id pub-id-type="pmcid">PMC6488959</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Leung</surname>
              <given-names>SF</given-names>
            </name>
            <name name-style="western">
              <surname>Ma</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Russell</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Enhancing motivation to change in eating disorders with an online self-help program</article-title>
          <source>Int J Ment Health Nurs</source>
          <year>2013</year>
          <month>08</month>
          <volume>22</volume>
          <issue>4</issue>
          <fpage>329</fpage>
          <lpage>339</lpage>
          <pub-id pub-id-type="doi">10.1111/j.1447-0349.2012.00870.x</pub-id>
          <pub-id pub-id-type="medline">22882349</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ondersma</surname>
              <given-names>SJ</given-names>
            </name>
            <name name-style="western">
              <surname>Svikis</surname>
              <given-names>DS</given-names>
            </name>
            <name name-style="western">
              <surname>Schuster</surname>
              <given-names>CR</given-names>
            </name>
          </person-group>
          <article-title>Computer-based brief intervention a randomized trial with postpartum women</article-title>
          <source>Am J Prev Med</source>
          <year>2007</year>
          <month>03</month>
          <volume>32</volume>
          <issue>3</issue>
          <fpage>231</fpage>
          <lpage>238</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/17236741"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.amepre.2006.11.003</pub-id>
          <pub-id pub-id-type="medline">17236741</pub-id>
          <pub-id pub-id-type="pii">S0749-3797(06)00494-6</pub-id>
          <pub-id pub-id-type="pmcid">PMC1858656</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>He</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Basar</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Wiers</surname>
              <given-names>RW</given-names>
            </name>
            <name name-style="western">
              <surname>Antheunis</surname>
              <given-names>ML</given-names>
            </name>
            <name name-style="western">
              <surname>Krahmer</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Can chatbots help to motivate smoking cessation? A study on the effectiveness of motivational interviewing on engagement and therapeutic alliance</article-title>
          <source>BMC Public Health</source>
          <year>2022</year>
          <month>04</month>
          <day>12</day>
          <volume>22</volume>
          <issue>1</issue>
          <fpage>726</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcpublichealth.biomedcentral.com/articles/10.1186/s12889-022-13115-x"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12889-022-13115-x</pub-id>
          <pub-id pub-id-type="medline">35413887</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12889-022-13115-x</pub-id>
          <pub-id pub-id-type="pmcid">PMC9003955</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Miller</surname>
              <given-names>WR</given-names>
            </name>
            <name name-style="western">
              <surname>Rollnick</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <source>Motivational Interviewing: Helping People Change, Third Edition</source>
          <year>2012</year>
          <publisher-loc>New York, NY</publisher-loc>
          <publisher-name>Guilford Press</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Erazo</surname>
              <given-names>WS</given-names>
            </name>
            <name name-style="western">
              <surname>Guerrero</surname>
              <given-names>GP</given-names>
            </name>
            <name name-style="western">
              <surname>Betancourt</surname>
              <given-names>CC</given-names>
            </name>
            <name name-style="western">
              <surname>Salazar</surname>
              <given-names>IS</given-names>
            </name>
          </person-group>
          <article-title>Chatbot implementation to collect data on possible COVID-19 cases and release the pressure on the primary health care system</article-title>
          <year>2020</year>
          <conf-name>11th IEEE Annual Information Technology, Electronics and Mobile Communication Conference (IEMCON)</conf-name>
          <conf-date>November 4-7, 2020</conf-date>
          <conf-loc>Virtual</conf-loc>
          <fpage>0302</fpage>
          <lpage>0307</lpage>
          <pub-id pub-id-type="doi">10.1109/iemcon51383.2020.9284846</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Athota</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Shukla</surname>
              <given-names>VK</given-names>
            </name>
            <name name-style="western">
              <surname>Pandey</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Rana</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Chatbot for healthcare system using artificial intelligence</article-title>
          <year>2020</year>
          <conf-name>8th International Conference on Reliability, Infocom Technologies and Optimization (Trends and Future Directions) (ICRITO)</conf-name>
          <conf-date>June 4-5, 2020</conf-date>
          <conf-loc>Noida, India</conf-loc>
          <fpage>619</fpage>
          <lpage>622</lpage>
          <pub-id pub-id-type="doi">10.1109/icrito48877.2020.9197833</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Habib</surname>
              <given-names>FA</given-names>
            </name>
            <name name-style="western">
              <surname>Shakil</surname>
              <given-names>GS</given-names>
            </name>
            <name name-style="western">
              <surname>Mohd. Iqbal</surname>
              <given-names>SS</given-names>
            </name>
            <name name-style="western">
              <surname>Sajid</surname>
              <given-names>STA</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Goyal</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Chaturvedi</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Nagar</surname>
              <given-names>AK</given-names>
            </name>
            <name name-style="western">
              <surname>Purohit</surname>
              <given-names>SD</given-names>
            </name>
          </person-group>
          <article-title>Self-diagnosis medical chatbot using artificial intelligence</article-title>
          <source>Proceedings of Second International Conference on Smart Energy and Communication</source>
          <year>2021</year>
          <publisher-loc>Singapore</publisher-loc>
          <publisher-name>Springer</publisher-name>
          <fpage>587</fpage>
          <lpage>593</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fadhil</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Gabrielli</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Addressing challenges in promoting healthy lifestyles: the AI-chatbot approach</article-title>
          <source>Proceedings of the 11th EAI International Conference on Pervasive Computing Technologies for Healthcare</source>
          <year>2017</year>
          <publisher-loc>New York, NY</publisher-loc>
          <publisher-name>Association for Computing Machinery</publisher-name>
          <fpage>261</fpage>
          <lpage>265</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fadhil</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <source>arXiv:1803.06000 [cs.CY] Preprint posted online 2018. [doi: 10.48550/arXiv.1803.06000]</source>
          <year>2021</year>
          <pub-id pub-id-type="doi">10.48550/ar</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>MacDonald</surname>
              <given-names>NE</given-names>
            </name>
            <collab>SAGE Working Group on Vaccine Hesitancy</collab>
          </person-group>
          <article-title>Vaccine hesitancy: definition, scope and determinants</article-title>
          <source>Vaccine</source>
          <year>2015</year>
          <month>08</month>
          <day>14</day>
          <volume>33</volume>
          <issue>34</issue>
          <fpage>4161</fpage>
          <lpage>4164</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0264-410X(15)00500-9"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.vaccine.2015.04.036</pub-id>
          <pub-id pub-id-type="medline">25896383</pub-id>
          <pub-id pub-id-type="pii">S0264-410X(15)00500-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gottlieb</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Dyer</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Information and disinformation: social media in the COVID-19 crisis</article-title>
          <source>Acad Emerg Med</source>
          <year>2020</year>
          <month>07</month>
          <day>24</day>
          <volume>27</volume>
          <issue>7</issue>
          <fpage>640</fpage>
          <lpage>641</lpage>
          <pub-id pub-id-type="doi">10.1111/acem.14036</pub-id>
          <pub-id pub-id-type="medline">32474977</pub-id>
          <pub-id pub-id-type="pmcid">PMC7300599</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gisondi</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Barber</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Faust</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Raja</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Strehlow</surname>
              <given-names>MC</given-names>
            </name>
            <name name-style="western">
              <surname>Westafer</surname>
              <given-names>LM</given-names>
            </name>
            <name name-style="western">
              <surname>Gottlieb</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>A deadly infodemic: social media and the power of COVID-19 misinformation</article-title>
          <source>J Med Internet Res</source>
          <year>2022</year>
          <month>2</month>
          <day>1</day>
          <volume>24</volume>
          <issue>2</issue>
          <fpage>e35552</fpage>
          <pub-id pub-id-type="doi">10.2196/35552</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kemp</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Digital 2021: global overview report</article-title>
          <source>DataReportal</source>
          <year>2021</year>
          <month>1</month>
          <day>27</day>
          <access-date>2023-07-20</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://datareportal.com/reports/digital-2021-global-overview-report">https://datareportal.com/reports/digital-2021-global-overview-report</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gottfried</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Shearer</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>News use across social media platforms 2016</article-title>
          <source>Pew Research Center</source>
          <year>2016</year>
          <month>5</month>
          <day>26</day>
          <access-date>2022-07-22</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.pewr search.org/journalism/2016/05/26/news-use-across-social-media-platforms-2016/">https://www.pewr search.org/journalism/2016/05/26/news-use-across-social-media-platforms-2016/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Suarez-Lledo</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Alvarez-Galvez</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Prevalence of health misinformation on social media: systematic review</article-title>
          <source>J Med Internet Res</source>
          <year>2021</year>
          <month>01</month>
          <day>20</day>
          <volume>23</volume>
          <issue>1</issue>
          <fpage>e17187</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2021/1/e17187/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/17187</pub-id>
          <pub-id pub-id-type="medline">33470931</pub-id>
          <pub-id pub-id-type="pii">v23i1e17187</pub-id>
          <pub-id pub-id-type="pmcid">PMC7857950</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="web">
          <article-title>Chat marketing made easy with ManyChat</article-title>
          <source>ManyChat</source>
          <access-date>2022-08-30</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://manychat.com/">https://manychat.com/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Albright</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Adam</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Serri</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Bleeker</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Goldman</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Harnessing the power of conversations with virtual humans to change health behaviors</article-title>
          <source>Mhealth</source>
          <year>2016</year>
          <month>11</month>
          <day>28</day>
          <volume>2</volume>
          <fpage>44</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/28293614"/>
          </comment>
          <pub-id pub-id-type="doi">10.21037/mhealth.2016.11.02</pub-id>
          <pub-id pub-id-type="medline">28293614</pub-id>
          <pub-id pub-id-type="pii">mh-02-2016.11.02</pub-id>
          <pub-id pub-id-type="pmcid">PMC5344154</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Bickmore</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Medical shared decision making with a virtual agent</article-title>
          <year>2018</year>
          <conf-name>IVA '18: 18th International Conference on Intelligent Virtual Agents</conf-name>
          <conf-date>November 5-8, 2018</conf-date>
          <conf-loc>Sydney, Australia</conf-loc>
          <fpage>113</fpage>
          <lpage>118</lpage>
          <pub-id pub-id-type="doi">10.1145/3267851.3267883</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alghamdi</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Alnanih</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Chatbot design for a healthy life to celiac patients: a study according to a new behavior change model</article-title>
          <source>IJACSA</source>
          <year>2021</year>
          <volume>12</volume>
          <issue>10</issue>
          <fpage>698</fpage>
          <lpage>707</lpage>
          <pub-id pub-id-type="doi">10.14569/ijacsa.2021.0121077</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mandayam</surname>
              <given-names>AU</given-names>
            </name>
            <name name-style="western">
              <surname>Siddesha</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Niranjan</surname>
              <given-names>SK</given-names>
            </name>
          </person-group>
          <article-title>Intelligent conversational model for mental health wellness</article-title>
          <source>TURCOMAT</source>
          <year>2022</year>
          <volume>13</volume>
          <issue>2</issue>
          <fpage>1008</fpage>
          <lpage>1017</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schulman</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Bickmore</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Sidner</surname>
              <given-names>CL</given-names>
            </name>
          </person-group>
          <article-title>An intelligent conversational agent for promoting long-term health behavior change using motivational interviewing</article-title>
          <year>2011</year>
          <conf-name>2011 AAAI Spring Symposium</conf-name>
          <conf-date>March 21–23, 2011</conf-date>
          <conf-loc>Palo Alto, CA</conf-loc>
          <fpage>61</fpage>
          <lpage>64</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="web">
          <article-title>Conversation design</article-title>
          <source>Google Developers</source>
          <access-date>2022-08-19</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://developers.google.com/assistant/conversation-design/welcome">https://developers.google.com/assistant/conversation-design/welcome</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="web">
          <article-title>The conversation design process: how to design chatbot dialogue</article-title>
          <source>KeyReply</source>
          <year>2021</year>
          <month>2</month>
          <day>4</day>
          <access-date>2022-02-11</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://keyreply com/blog/chatbots-conversational-design/">https://keyreply com/blog/chatbots-conversational-design/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sai Sharath</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Banafsheh</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Conversational question answering over knowledge base using chatbot framework</article-title>
          <year>2021</year>
          <conf-name>IEEE 15th International Conference on Semantic Computing (ICSC)</conf-name>
          <conf-date>January 27-29, 2021</conf-date>
          <conf-loc>Laguna Hills, CA</conf-loc>
          <fpage>84</fpage>
          <lpage>85</lpage>
          <pub-id pub-id-type="doi">10.1109/icsc50631.2021.00020</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pears</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Henderson</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Konstantinidis</surname>
              <given-names>STh</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Mantas</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Stoicu-Tivadar</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Chronaki</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Hasman</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Weber</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Gallos</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Crişan-Vida</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Zoulias</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Chirila</surname>
              <given-names>OS</given-names>
            </name>
          </person-group>
          <article-title>Repurposing case-based learning to a conversational agent for healthcare cybersecurity</article-title>
          <source>Studies in Health Technology and Informatics</source>
          <year>2021</year>
          <publisher-loc>Amsterdam, the Netherlands</publisher-loc>
          <publisher-name>IOS Press</publisher-name>
          <fpage>0348</fpage>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="web">
          <article-title>COVID-19 question and answers hub</article-title>
          <source>World Health Organization</source>
          <access-date>2022-08-09</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.who.int/emergencies/diseases/novel-coronavirus-2019/question-and-answers-hub">https://www.who.int/emergencies/diseases/novel-coronavirus-2019/question-and-answers-hub</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="web">
          <article-title>COVID-19 vaccination</article-title>
          <source>Centers for Disease Control and Prevention</source>
          <year>2022</year>
          <month>5</month>
          <day>17</day>
          <access-date>2022-08-09</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.cdc. ov/coronavirus/2019-ncov/vaccines/index.html">https://www.cdc. ov/coronavirus/2019-ncov/vaccines/index.html</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gross</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Schachner</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Hasl</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kohlbrenner</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Clarenbach</surname>
              <given-names>CF</given-names>
            </name>
            <name name-style="western">
              <surname>Wangenheim</surname>
              <given-names>FV</given-names>
            </name>
            <name name-style="western">
              <surname>Kowatsch</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Personalization of conversational agent-patient interaction styles for chronic disease management: two consecutive cross-sectional questionnaire studies</article-title>
          <source>J Med Internet Res</source>
          <year>2021</year>
          <month>05</month>
          <day>26</day>
          <volume>23</volume>
          <issue>5</issue>
          <fpage>e26643</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2021/5/e26643/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/26643</pub-id>
          <pub-id pub-id-type="medline">33913814</pub-id>
          <pub-id pub-id-type="pii">v23i5e26643</pub-id>
          <pub-id pub-id-type="pmcid">PMC8190651</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="web">
          <article-title>Chatbot personalization: why it’s needed to improve customer experience</article-title>
          <source>Instapage</source>
          <year>2019</year>
          <month>1</month>
          <day>3</day>
          <access-date>2022-08-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://instapage.com/blog/chatbot-personalization">https://instapage.com/blog/chatbot-personalization</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pearl</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Everything you ever wanted to know about conversation design but were afraid to ask</article-title>
          <source>Google</source>
          <year>2019</year>
          <month>4</month>
          <day>12</day>
          <access-date>2023-08-19</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.youtube.com/watch?v=vafh50qmWMM">https://www.youtube.com/watch?v=vafh50qmWMM</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gabarda</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Butterworth</surname>
              <given-names>SW</given-names>
            </name>
          </person-group>
          <article-title>Using best practices to address COVID-19 vaccine hesitancy: the case for the motivational interviewing approach</article-title>
          <source>Health Promot Pract</source>
          <year>2021</year>
          <month>09</month>
          <day>08</day>
          <volume>22</volume>
          <issue>5</issue>
          <fpage>611</fpage>
          <lpage>615</lpage>
          <pub-id pub-id-type="doi">10.1177/15248399211016463</pub-id>
          <pub-id pub-id-type="medline">33966471</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Friederichs</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bolman</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Oenema</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Guyaux</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Lechner</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Motivational interviewing in a web-based physical activity intervention with an avatar: randomized controlled trial</article-title>
          <source>J Med Internet Res</source>
          <year>2014</year>
          <month>02</month>
          <day>13</day>
          <volume>16</volume>
          <issue>2</issue>
          <fpage>e48</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2014/2/e48/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.2974</pub-id>
          <pub-id pub-id-type="medline">24550153</pub-id>
          <pub-id pub-id-type="pii">v16i2e48</pub-id>
          <pub-id pub-id-type="pmcid">PMC3936285</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Holmes</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Moorhead</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Bond</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Zheng</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Coates</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Mctear</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Usability testing of a healthcare chatbot: can we use conventional methods to assess conversational user interfaces?</article-title>
          <year>2019</year>
          <conf-name>ECCE'19: 31st European Conference on Cognitive Ergonomics</conf-name>
          <conf-date>September 10-13, 2019</conf-date>
          <conf-loc>Belfast, UK</conf-loc>
          <fpage>207</fpage>
          <lpage>214</lpage>
          <pub-id pub-id-type="doi">10.1145/3335082.3335094</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Brooke</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>SUS: a ‘quick and dirty’ usability scale</article-title>
          <source>Usability Evaluation In Industry</source>
          <year>1996</year>
          <publisher-loc>Boca Raton, FL</publisher-loc>
          <publisher-name>CRC Press</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Miller</surname>
              <given-names>WR</given-names>
            </name>
            <name name-style="western">
              <surname>Rollnick</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <source>Motivational Interviewing: Preparing People for Change, 2nd ed</source>
          <year>2002</year>
          <publisher-loc>New York, NY</publisher-loc>
          <publisher-name>Guilford Press</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nißen</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rüegger</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Stieger</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Flückiger</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Allemand</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>V Wangenheim</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Kowatsch</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>The effects of health care chatbot personas with different social roles on the client-chatbot bond and usage intentions: development of a design codebook and web-based study</article-title>
          <source>J Med Internet Res</source>
          <year>2022</year>
          <month>04</month>
          <day>27</day>
          <volume>24</volume>
          <issue>4</issue>
          <fpage>e32630</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2022/4/e32630/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/32630</pub-id>
          <pub-id pub-id-type="medline">35475761</pub-id>
          <pub-id pub-id-type="pii">v24i4e32630</pub-id>
          <pub-id pub-id-type="pmcid">PMC9096656</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ahmad</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Siemon</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Gnewuch</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Robra-Bissantz</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Designing personality-adaptive conversational agents for mental health care</article-title>
          <source>Inf Syst Front</source>
          <year>2022</year>
          <month>03</month>
          <day>02</day>
          <volume>24</volume>
          <issue>3</issue>
          <fpage>923</fpage>
          <lpage>943</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/35250365"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s10796-022-10254-9</pub-id>
          <pub-id pub-id-type="medline">35250365</pub-id>
          <pub-id pub-id-type="pii">10254</pub-id>
          <pub-id pub-id-type="pmcid">PMC8889396</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Yamashita</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Huang</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Designing a chatbot as a mediator for promoting deep self-disclosure to a real mental health professional</article-title>
          <source>Proc ACM Hum-Comput Interact</source>
          <year>2020</year>
          <month>05</month>
          <day>29</day>
          <volume>4</volume>
          <issue>CSCW1</issue>
          <fpage>1</fpage>
          <lpage>27</lpage>
          <pub-id pub-id-type="doi">10.1145/3392836</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref49">
        <label>49</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nurmi</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Knittle</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Ginchev</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Khattak</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Helf</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Zwickl</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Castellano-Tejedor</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Lusilla-Palacios</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Costa-Requena</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ravaja</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Haukkala</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Engaging users in the behavior change process with digitalized motivational interviewing and gamification: development and feasibility testing of the precious app</article-title>
          <source>JMIR Mhealth Uhealth</source>
          <year>2020</year>
          <month>1</month>
          <day>30</day>
          <volume>8</volume>
          <issue>1</issue>
          <fpage>e12884</fpage>
          <pub-id pub-id-type="doi">10.2196/12884</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref50">
        <label>50</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rahman</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Rahman</surname>
              <given-names>MR</given-names>
            </name>
            <name name-style="western">
              <surname>Tripto</surname>
              <given-names>NI</given-names>
            </name>
          </person-group>
          <article-title>Adolescentbot: understanding opportunities for chatbots in combating adolescent sexual and reproductive health problems in Bangladesh</article-title>
          <year>2021</year>
          <conf-name>CHI '21: CHI Conference on Human Factors in Computing Systems</conf-name>
          <conf-date>May 8-13, 2021</conf-date>
          <conf-loc>Yokohama, Japan</conf-loc>
          <pub-id pub-id-type="doi">10.1145/3411764.3445694</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref51">
        <label>51</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hall</surname>
              <given-names>MJ</given-names>
            </name>
          </person-group>
          <article-title>Personas: designing personalized, learner-centric experiences</article-title>
          <source>Association for Talent Development</source>
          <year>2018</year>
          <month>10</month>
          <day>31</day>
          <access-date>2023-07-20</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.td.org/insights/personas-designing-personalized-learner-centric-experiences">https://www.td.org/insights/personas-designing-personalized-learner-centric-experiences</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref52">
        <label>52</label>
        <nlm-citation citation-type="web">
          <article-title>Conversational actions</article-title>
          <source>Google Developers</source>
          <access-date>2022-09-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://developers.google.com/assistant/conversational">https://developers.google.com/assistant/conversational</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref53">
        <label>53</label>
        <nlm-citation citation-type="web">
          <article-title>Master the art of creating winning AI Assistants!</article-title>
          <source>Conversation Design Institute</source>
          <access-date>2022-09-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.conversationdesigninstitute.com/">https://www.conversationdesigninstitute.com/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref54">
        <label>54</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Almusharraf</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Rose</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Selby</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Engaging unmotivated smokers to move toward quitting: design of motivational interviewing-based chatbot through iterative interactions</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <month>11</month>
          <day>03</day>
          <volume>22</volume>
          <issue>11</issue>
          <fpage>e20251</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/11/e20251/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/20251</pub-id>
          <pub-id pub-id-type="medline">33141095</pub-id>
          <pub-id pub-id-type="pii">v22i11e20251</pub-id>
          <pub-id pub-id-type="pmcid">PMC7671850</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref55">
        <label>55</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Prochaska</surname>
              <given-names>JJ</given-names>
            </name>
            <name name-style="western">
              <surname>Vogel</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Chieng</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kendra</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Baiocchi</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Pajarito</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Robinson</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>A therapeutic relational agent for reducing problematic substance use (Woebot): development and usability study</article-title>
          <source>J Med Internet Res</source>
          <year>2021</year>
          <month>03</month>
          <day>23</day>
          <volume>23</volume>
          <issue>3</issue>
          <fpage>e24850</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2021/3/e24850/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/24850</pub-id>
          <pub-id pub-id-type="medline">33755028</pub-id>
          <pub-id pub-id-type="pii">v23i3e24850</pub-id>
          <pub-id pub-id-type="pmcid">PMC8074987</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref56">
        <label>56</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Altay</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Hacquin</surname>
              <given-names>A-S</given-names>
            </name>
            <name name-style="western">
              <surname>Chevallier</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Mercier</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Information delivered by a chatbot has a positive impact on COVID-19 vaccines attitudes and intentions</article-title>
          <source>PsyArXiv Preprint posted online 2021. [doi: 10.31234/os .io/eb2gt]</source>
          <year>2021</year>
          <pub-id pub-id-type="doi">10.31234/osf.io/eb2gt</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
