diff --git a/references.bib b/references.bib index 7204382..414eb9b 100755 --- a/references.bib +++ b/references.bib @@ -659,6 +659,16 @@ learners are different, and (b) that they change as they learn. But}, url = {http://misq.org/perceived-usefulness-perceived-ease-of-use-and-user-acceptance-of-information-technology.html}, } +@article{likert1932, + title = {A Technique for the Measurement of Attitudes}, + author = {Likert, Rensis}, + journal = {Archives of Psychology}, + year = {1932}, + volume = {140}, + pages = {1--55}, + url = {https://psycnet.apa.org/record/1933-01885-001} +} + @Book{Coe2025, editor = {Robert Coe and Michael Waring and Larry Hedges and Laura Day Ashley}, publisher = {SAGE Publications}, diff --git a/slides.tex b/slides.tex index d4ae9ce..c1ca46e 100644 --- a/slides.tex +++ b/slides.tex @@ -166,109 +166,123 @@ \section{How I plan to address this space} -\begin{frame} - \frametitle{How am I addressing the gap} - \todo{Talk about Gidget and offer scaffolding tools, then talk about how we - can create a tool to aid intervention} - I plan to create an intervention wherein I give students access to an AI tool - that takes elements from previous interventions that align with the principles - of improving self-efficacy in students, and combine it with more recent - breakthroughs in LLM based technologies to allow the intervention to be used - in a wider variety of programming environments and contexts. +%\begin{frame} +% \frametitle{How am I addressing the gap} +% \todo{Talk about Gidget and offer scaffolding tools, then talk about how we +% can create a tool to aid intervention} +% I plan to create an intervention wherein I give students access to an AI tool +% that takes elements from previous interventions that align with the principles +% of improving self-efficacy in students, and combine it with more recent +% breakthroughs in LLM based technologies to allow the intervention to be used +% in a wider variety of programming environments and contexts. +% +% I then plan to evaluate its impact on self-efficacy and potentially other +% outcomes. \todo{Figure out what those outcomes are.} +%\end{frame} - I then plan to evaluate its impact on self-efficacy and potentially other - outcomes. \todo{Figure out what those outcomes are.} -\end{frame} +%\begin{frame} +% \frametitle{The Intervention} +% \todo{Talk about methodology and the type of intervention that's taking place. +% Why this method over other methods etc} +% +% This artefact attempts to replicate the channels of self-efficacy improvement +% that one on one support from an instructor, by targeting these factors in +% Bandura's theory \todo{cite}: +%\begin{itemize} +% \item Mastery experiences (Experiencing more success in coding tasks). +% \item Verbal encouragement. +% \item Guided support. +%\end{itemize} +% \todo{Check the previous itemisation isn't mistaken} +% +% The agent will use techniques such as personification \todo{cite gidget} and +% an AI models ability to translate errors into more accessible descriptions +% replicate an instructors ability to help a student understand knowledge they +% may be missing in terms of the language of a compiler. The model will also +% focus on guiding the student towards the solution in a structured manner such +% as in \todo{cite structured debugging intervention paper}. +% +% The goal with this artefact is to as closely as possible replicate the +% experience and self-efficacy improvements that come with 1 on 1 tuition +% \todo{cite}, so that these benefits can be scaled up. And help seeking +% behaviours be made more available. +%\end{frame} \begin{frame} \frametitle{The Intervention} - \todo{Talk about methodology and the type of intervention that's taking place. - Why this method over other methods etc} + I plan to create an artefact that uses scaffolding and personification in an + attempt to improve students self-efficacy \cite{Bandura1977} and reduce + cognitive load \cite{Sweller1988}. + \newline - This artefact attempts to replicate the channels of self-efficacy improvement - that one on one support from an instructor, by targeting these factors in - Bandura's theory \todo{cite}: + This will be done by walking a student through any issues a + real programming environment can throw at them with the use of an AI agent + based avatar, which will break down the process of solving debugging issues in + their own real world programming environment. +\end{frame} + +\begin{frame} + \frametitle{Methodology} + Due to the nature of using a software tool to improve outcomes, an + intervention based experiment becomes necessary \cite[p.~242]{Coe2025}. The + projects' reliance on AI should also adhere to guidelines such as + CONSORT-AI \cite{Liu2020} in order to remain reproducible and transparent. + \newline + + If ethical issues surrounding the withholding of resources from some students + that were made available to others even at random wasn't a worry, a random + selection of student participants would be the best approach to reduce bias + \cite[p.~245]{Coe2025}. + \newline + + To counter this issue will I will be measuring results from a cohort before + the intervention has been fully developed and comparing results to a cohort + that has then since gained access to the artefact in a longitudinal study + \cite[p.~224]{Coe2025}. + \begin{itemize} + \item Cohort A -- Pre-intervention Cohort + \item Cohort B -- intervention Cohort + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Data Collection} + %TODO come back to this + Self-efficacy is a psychological self-perception and as such must be collected + from communicating with the participants. + \newline + + There are however several methods we can use to quantify qualitative data + through tools like Likert scales \cite{likert1932} and thematic analysis + \cite{Braun2006} of feedback. + \newline + + Both of these methods require careful planning to avoid the influence of bias. + I've interacted with them before in papers I've contributed to in the past + \cite{Mitchell2021, Mitchell2022}, and I am confident in the GA's ability to + guide me through the process of making sure I collect this data correctly. +\end{frame} + +\section{Technology stacks and ethical considerations} + +%\begin{frame} +% \frametitle{Key challenges in the study} +%\end{frame} + +\begin{frame} + \frametitle{AI Ethical Concerns} + This research immediately falls into at least medium risk in Falmouth + University's ethics policy as it involves human participants. Due to the + nature of interacting with generative AI addition measures will be necessary. \begin{itemize} - \item Mastery experiences (Experiencing more success in coding tasks). - \item Verbal encouragement. - \item Guided support. -\end{itemize} - \todo{Check the previous itemisation isn't mistaken} - - The agent will use techniques such as personification \todo{cite gidget} and - an AI models ability to translate errors into more accessible descriptions - replicate an instructors ability to help a student understand knowledge they - may be missing in terms of the language of a compiler. The model will also - focus on guiding the student towards the solution in a structured manner such - as in \todo{cite structured debugging intervention paper}. - - The goal with this artefact is to as closely as possible replicate the - experience and self-efficacy improvements that come with 1 on 1 tuition - \todo{cite}, so that these benefits can be scaled up. And help seeking - behaviours be made more available. -\end{frame} - -\begin{frame} - \frametitle{Key influences in this field and area of research.} - \todo{Talk about the psychology and mikes previous paper bandura. gidget and - some scaffolding intervention if it's not all covered in the previous slide} -\end{frame} - -\begin{frame} - \frametitle{What is the variable. What does better look like} - \todo{what research methods will analyse this} - \todo{Students drop out less, produce higher quality code? What do other - interventionist studies look at?} - \todo{There are a few outcomes we can try to measure. We can use likart style - surveys and attempt pre-post intervention comparisons} - - The study plans to measure changes in the students self-efficacy and their - self perception of their coding ability before and after intervention. - - The study falls into an educational intervention style methodology for which - the best style of study in regard to bias is usually random - assignment \todo{cite research methods and methodologies in education}. - However, due to the ethical concerns of depriving only part of a cohort of a - tool that may be beneficial to them without an alternative equally powerful - tool as an alternative, a longitudinal year of entry based cohort study could - be used as an alternative, albeit with a higher potential for bias.\todo{cite - research methods and methodologies in education different page} -\end{frame} - -\section{Challenges and ethical considerations} - -\begin{frame} - \frametitle{Key challenges in the study} - \todo{Talk about the time to develop the software and more about the study - structure and issues with pedagogical research in general} -\end{frame} - -\begin{frame} - \frametitle{main ethical concerns} - -\begin{itemize} - \item \todo{This research should be limited to adults as gen ai and children = bad.} - \item \todo{Making the AI friendlier and having it say something outside my control} - \item \todo{Mention potential for bias in the current data collection} - \todo{Ethical AI models should be a requirement} + \item We should avoid giving AI access to children and vulnerable people. + \item We should make sure to use ethically trained AI models where possible. + \item Furthermore, we should follow CONSORT-AI \cite{Liu2020} to keep this + research transparent and reproducible. \end{itemize} \end{frame} -\begin{frame} - \frametitle{Key challenges in this area} -\begin{itemize} - \item \todo{First stage programming generally has a high drop out rate because - overcoming programming problems can be distressing without a degree of - self-efficacy.} - \item \todo{Having a guide help students learn through the process one on one - is a time intensive process for staff and fairly cost inefficient in terms - of time spent with educators.} - \item \todo{This tool is also an attempt to scale the amount of human support - a student can receive, outside the classroom} -\end{itemize} -\end{frame} - \section{Me, My Skills, And why I want to study at Falmouth University} % 1 4 9? 10? @@ -294,9 +308,9 @@ Falmouth University previously: \begin{itemize} \item Student Perspectives on the Purpose of Peer Evaluation During Group - Game Development Projects \cite{Mitchell2021} + Game Development Projects \cite{Mitchell2021}. \item An Exploratory Analysis of Student Experiences with Peer Evaluation in - Group Game Development Projects \cite{Mitchell2022} + Group Game Development Projects \cite{Mitchell2022}. \end{itemize} \end{frame}